{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "dgBTU57OWYI0"
   },
   "source": [
    "# Install Library\n",
    "\n",
    "[RDKit ](https://github.com/rdkit/rdkit)\n",
    "\n",
    "[DGL](https://github.com/dmlc/dgl/)\n",
    "\n",
    "[DGL-LifeSci](https://github.com/awslabs/dgl-lifesci)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "EOF1QxeqhajG"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install rdkit-pypi\n",
    "!pip install dgllife\n",
    "!pip install --pre dgl-cu113 dglgo -f https://data.dgl.ai/wheels-test/repo.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xtojkovzWYI2"
   },
   "source": [
    "# Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ep6p1t9b8Q8g"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import dgl\n",
    "import sys\n",
    "import torch\n",
    "import random\n",
    "import cv2\n",
    "import statistics\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.optim as optim\n",
    "\n",
    "from rdkit.Chem import AllChem\n",
    "from rdkit import Chem\n",
    "from rdkit import DataStructs\n",
    "\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.callbacks import  History\n",
    "from dgllife.utils import smiles_to_bigraph, CanonicalAtomFeaturizer, AttentiveFPAtomFeaturizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from utils.general import DATASET, get_dataset, separate_active_and_inactive_data, get_embedding_vector_class, count_lablel,data_generator\n",
    "from utils.gcn_pre_trained import get_sider_model\n",
    "from utils.special_functions import is_Membership\n",
    "\n",
    "from model.heterogeneous_siamese_sider import siamese_model_attentiveFp_sider\n",
    "\n",
    "device = torch.device('cpu' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "1RVgRpTmQ5rp",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "cache_path_tox21='./tox21_dglgraph.bin'\n",
    "\n",
    "df_tox21 = get_dataset(\"tox21\")\n",
    "ids = df_tox21['mol_id']\n",
    "\n",
    "df_tox21 = df_tox21.drop(columns=['mol_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "cache_path_sider='./sider_dglgraph.bin'\n",
    "\n",
    "df = get_dataset(\"sider\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "wr1GMh7H8Q8i"
   },
   "outputs": [],
   "source": [
    "tox21_tasks = df.columns.values[:12].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "IqdSnLd28Q8i"
   },
   "outputs": [],
   "source": [
    "tox21_smiles = np.array(df_tox21['smiles'])\n",
    "sider_smiles = np.array(df_sider['smiles'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "fmcg_s2H8Q8i"
   },
   "outputs": [],
   "source": [
    "subscriber = []\n",
    "for ts in tox21_smiles:\n",
    "    for ss in sider_smiles:\n",
    "        if ts == ss:\n",
    "            subscriber.append(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "Bcx5CfKe8Q8j",
    "outputId": "058955ca-d42a-4646-b391-e1fd49a60294"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CC(O)(P(=O)(O)O)P(=O)(O)O',\n",
       " 'C[N+](C)(C)CC(=O)[O-]',\n",
       " 'C[N+](C)(C)CCO',\n",
       " 'CC(=O)NO',\n",
       " 'CC(=O)OCC[N+](C)(C)C',\n",
       " 'CC(=O)[O-].[Na+]',\n",
       " 'CCCC(CCC)C(=O)O',\n",
       " 'Cl[Zn]Cl',\n",
       " 'CN(CCCl)CCCl',\n",
       " 'C[N+](C)(C)CCOC(=O)CCC(=O)OCC[N+](C)(C)C',\n",
       " 'C1N2CN3CN1CN(C2)C3',\n",
       " 'CCN(CC)C(=S)SSC(=S)N(CC)CC']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subscriber"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "6grIE_JeqkUZ"
   },
   "source": [
    "# Required functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "IzllOg474i99"
   },
   "outputs": [],
   "source": [
    "def create_dataset_with_gcn_case_study(dataset, class_embed_vector, GCN, tasks):\n",
    "    created_data = []\n",
    "    data = np.arange(len(tasks))\n",
    "    onehot_encoded = to_categorical(data)\n",
    "    for i, data in enumerate(dataset):\n",
    "        smiles, g, labels, mask = data\n",
    "        g = g.to(device)\n",
    "        g = dgl.add_self_loop(g)\n",
    "        graph_feats = g.ndata.pop('h')\n",
    "        embbed = GCN(g, graph_feats)\n",
    "        embbed = embbed.to('cpu')\n",
    "        embbed = embbed.detach().numpy()\n",
    "        for j, label in enumerate(labels):\n",
    "            a = (smiles, embbed, onehot_encoded[j], class_embed_vector[j], labels[j], tasks[j])\n",
    "            created_data.append(a)\n",
    "    print('Data created!!')\n",
    "    return created_data\n",
    "\n",
    "\n",
    "def create_dataset_with_gcn(dataset, subscriber, class_embed_vector, GCN, tasks, numberTask):\n",
    "\n",
    "    created_data = []\n",
    "    created_subscriber = []\n",
    "    data = np.arange(len(tasks))\n",
    "    onehot_encoded = to_categorical(data)\n",
    "\n",
    "    for i, data in enumerate(dataset):\n",
    "        smiles, g, label, mask = data\n",
    "#         g = g.to(device)\n",
    "        g = dgl.add_self_loop(g)\n",
    "        graph_feats = g.ndata.pop('h')\n",
    "        embbed = GCN(g, graph_feats)\n",
    "        embbed = embbed.to('cpu')\n",
    "        embbed = embbed.detach().numpy()\n",
    "        a = (smiles, embbed, onehot_encoded[numberTask], class_embed_vector[numberTask], label, tasks[numberTask])\n",
    "        if smiles in subscriber:\n",
    "            created_subscriber.append(data)\n",
    "        else:\n",
    "            created_data.append(a)\n",
    "    print('Data created!!')\n",
    "    return created_data, created_subscriber\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "SzamTlhQ8Q8k"
   },
   "source": [
    "# Calculation of embedded vectors for each class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "_2y-h9wu8Q8k",
    "outputId": "54d874d9-6ec6-4396-af7e-8992a7ac7170"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR-AR=> positive: 309 - negative: 6956\n",
      "NR-AR-LBD=> positive: 237 - negative: 6521\n",
      "NR-AhR=> positive: 768 - negative: 5781\n",
      "NR-Aromatase=> positive: 300 - negative: 5521\n",
      "NR-ER=> positive: 793 - negative: 5400\n",
      "NR-ER-LBD=> positive: 350 - negative: 6605\n",
      "NR-PPAR-gamma=> positive: 186 - negative: 6264\n",
      "SR-ARE=> positive: 942 - negative: 4890\n",
      "SR-ATAD5=> positive: 264 - negative: 6808\n",
      "SR-HSE=> positive: 372 - negative: 6095\n",
      "SR-MMP=> positive: 918 - negative: 4892\n",
      "SR-p53=> positive: 423 - negative: 6351\n"
     ]
    }
   ],
   "source": [
    "df_positive, df_negative = Separate_active_and_inactive_data(df_tox21, tox21_tasks)\n",
    "\n",
    "for i,d in enumerate(zip(df_positive,df_negative)):\n",
    "    print(f'{tox21_tasks[i]}=> positive: {len(d[0])} - negative: {len(d[1])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "G5Zv38xt8Q8k",
    "outputId": "c2e14bf2-54b2-4edc-b2a8-9baf18de9e6f",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6956\n",
      "Processing molecule 2000/6956\n",
      "Processing molecule 3000/6956\n",
      "Processing molecule 4000/6956\n",
      "Processing molecule 5000/6956\n",
      "Processing molecule 6000/6956\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6521\n",
      "Processing molecule 2000/6521\n",
      "Processing molecule 3000/6521\n",
      "Processing molecule 4000/6521\n",
      "Processing molecule 5000/6521\n",
      "Processing molecule 6000/6521\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5781\n",
      "Processing molecule 2000/5781\n",
      "Processing molecule 3000/5781\n",
      "Processing molecule 4000/5781\n",
      "Processing molecule 5000/5781\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5521\n",
      "Processing molecule 2000/5521\n",
      "Processing molecule 3000/5521\n",
      "Processing molecule 4000/5521\n",
      "Processing molecule 5000/5521\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5400\n",
      "Processing molecule 2000/5400\n",
      "Processing molecule 3000/5400\n",
      "Processing molecule 4000/5400\n",
      "Processing molecule 5000/5400\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6605\n",
      "Processing molecule 2000/6605\n",
      "Processing molecule 3000/6605\n",
      "Processing molecule 4000/6605\n",
      "Processing molecule 5000/6605\n",
      "Processing molecule 6000/6605\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6264\n",
      "Processing molecule 2000/6264\n",
      "Processing molecule 3000/6264\n",
      "Processing molecule 4000/6264\n",
      "Processing molecule 5000/6264\n",
      "Processing molecule 6000/6264\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/4890\n",
      "Processing molecule 2000/4890\n",
      "Processing molecule 3000/4890\n",
      "Processing molecule 4000/4890\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6808\n",
      "Processing molecule 2000/6808\n",
      "Processing molecule 3000/6808\n",
      "Processing molecule 4000/6808\n",
      "Processing molecule 5000/6808\n",
      "Processing molecule 6000/6808\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6095\n",
      "Processing molecule 2000/6095\n",
      "Processing molecule 3000/6095\n",
      "Processing molecule 4000/6095\n",
      "Processing molecule 5000/6095\n",
      "Processing molecule 6000/6095\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/4892\n",
      "Processing molecule 2000/4892\n",
      "Processing molecule 3000/4892\n",
      "Processing molecule 4000/4892\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6351\n",
      "Processing molecule 2000/6351\n",
      "Processing molecule 3000/6351\n",
      "Processing molecule 4000/6351\n",
      "Processing molecule 5000/6351\n",
      "Processing molecule 6000/6351\n"
     ]
    }
   ],
   "source": [
    "dataset_positive = [DATASET(d,smiles_to_bigraph, AttentiveFPAtomFeaturizer(), cache_file_path = cache_path) for d in df_positive]\n",
    "dataset_negative = [DATASET(d,smiles_to_bigraph, AttentiveFPAtomFeaturizer(), cache_file_path = cache_path) for d in df_negative]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "Kio5M0vt8Q8k",
    "outputId": "cc3f8dbb-dc87-4a3c-de30-224643fb3580"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class vector created!!\n"
     ]
    }
   ],
   "source": [
    "embed_class_tox21 = get_embedding_vector_class(dataset_positive, dataset_negative, subscriber, radius=2, size = 512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "zIKQi__XAcia"
   },
   "source": [
    "# Transfer Learning with BioAct-Het and AttentiveFp GCN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "lDS5UguKr_x_",
    "outputId": "da58be7e-197e-4838-f5f1-a0b2d6b87cb2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading GCN_attentivefp_SIDER_pre_trained.pth from https://data.dgl.ai/dgllife/pre_trained/gcn_attentivefp_sider.pth...\n",
      "Pretrained model loaded\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GCNPredictor(\n",
       "  (gnn): GCN(\n",
       "    (gnn_layers): ModuleList(\n",
       "      (0): GCNLayer(\n",
       "        (graph_conv): GraphConv(in=39, out=256, normalization=none, activation=<function relu at 0x000002844CF3C1F8>)\n",
       "        (dropout): Dropout(p=0.08333992387843633, inplace=False)\n",
       "        (bn_layer): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): GCNLayer(\n",
       "        (graph_conv): GraphConv(in=256, out=256, normalization=none, activation=<function relu at 0x000002844CF3C1F8>)\n",
       "        (dropout): Dropout(p=0.08333992387843633, inplace=False)\n",
       "        (bn_layer): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): GCNLayer(\n",
       "        (graph_conv): GraphConv(in=256, out=256, normalization=none, activation=<function relu at 0x000002844CF3C1F8>)\n",
       "        (dropout): Dropout(p=0.08333992387843633, inplace=False)\n",
       "        (bn_layer): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): GCNLayer(\n",
       "        (graph_conv): GraphConv(in=256, out=256, normalization=none, activation=<function relu at 0x000002844CF3C1F8>)\n",
       "        (dropout): Dropout(p=0.08333992387843633, inplace=False)\n",
       "        (bn_layer): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (readout): WeightedSumAndMax(\n",
       "    (weight_and_sum): WeightAndSum(\n",
       "      (atom_weighting): Sequential(\n",
       "        (0): Linear(in_features=256, out_features=1, bias=True)\n",
       "        (1): Sigmoid()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (predict): MLPPredictor(\n",
       "    (predict): Sequential(\n",
       "      (0): Dropout(p=0.08333992387843633, inplace=False)\n",
       "      (1): Linear(in_features=512, out_features=1024, bias=True)\n",
       "      (2): ReLU()\n",
       "      (3): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = 'GCN_attentivefp_SIDER'\n",
    "gcn_model = get_sider_model(model_name)\n",
    "gcn_model.eval()\n",
    "# gcn_model = gcn_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "kVwMzaZa8Q8l",
    "outputId": "c212caf3-43c2-416e-8c68-63c8a9fb2a01",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/7265\n",
      "Processing molecule 2000/7265\n",
      "Processing molecule 3000/7265\n",
      "Processing molecule 4000/7265\n",
      "Processing molecule 5000/7265\n",
      "Processing molecule 6000/7265\n",
      "Processing molecule 7000/7265\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6758\n",
      "Processing molecule 2000/6758\n",
      "Processing molecule 3000/6758\n",
      "Processing molecule 4000/6758\n",
      "Processing molecule 5000/6758\n",
      "Processing molecule 6000/6758\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6549\n",
      "Processing molecule 2000/6549\n",
      "Processing molecule 3000/6549\n",
      "Processing molecule 4000/6549\n",
      "Processing molecule 5000/6549\n",
      "Processing molecule 6000/6549\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5821\n",
      "Processing molecule 2000/5821\n",
      "Processing molecule 3000/5821\n",
      "Processing molecule 4000/5821\n",
      "Processing molecule 5000/5821\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6193\n",
      "Processing molecule 2000/6193\n",
      "Processing molecule 3000/6193\n",
      "Processing molecule 4000/6193\n",
      "Processing molecule 5000/6193\n",
      "Processing molecule 6000/6193\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6955\n",
      "Processing molecule 2000/6955\n",
      "Processing molecule 3000/6955\n",
      "Processing molecule 4000/6955\n",
      "Processing molecule 5000/6955\n",
      "Processing molecule 6000/6955\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6450\n",
      "Processing molecule 2000/6450\n",
      "Processing molecule 3000/6450\n",
      "Processing molecule 4000/6450\n",
      "Processing molecule 5000/6450\n",
      "Processing molecule 6000/6450\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5832\n",
      "Processing molecule 2000/5832\n",
      "Processing molecule 3000/5832\n",
      "Processing molecule 4000/5832\n",
      "Processing molecule 5000/5832\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/7072\n",
      "Processing molecule 2000/7072\n",
      "Processing molecule 3000/7072\n",
      "Processing molecule 4000/7072\n",
      "Processing molecule 5000/7072\n",
      "Processing molecule 6000/7072\n",
      "Processing molecule 7000/7072\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6467\n",
      "Processing molecule 2000/6467\n",
      "Processing molecule 3000/6467\n",
      "Processing molecule 4000/6467\n",
      "Processing molecule 5000/6467\n",
      "Processing molecule 6000/6467\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5810\n",
      "Processing molecule 2000/5810\n",
      "Processing molecule 3000/5810\n",
      "Processing molecule 4000/5810\n",
      "Processing molecule 5000/5810\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6774\n",
      "Processing molecule 2000/6774\n",
      "Processing molecule 3000/6774\n",
      "Processing molecule 4000/6774\n",
      "Processing molecule 5000/6774\n",
      "Processing molecule 6000/6774\n",
      "Data created!!\n"
     ]
    }
   ],
   "source": [
    "data_ds = []\n",
    "subscriber_data_ds = []\n",
    "for i, task in  enumerate(tox21_tasks):\n",
    "    a = df_tox21[['smiles' , task]]\n",
    "    a = a.dropna()\n",
    "    ds = DATASET(a,smiles_to_bigraph, AttentiveFPAtomFeaturizer(), cache_file_path = cache_path_sider) \n",
    "    data, subscriber_data = create_dataset_with_gcn(ds, subscriber, embed_class_tox21, gcn_model, tox21_tasks, i)\n",
    "    for d in data:\n",
    "        data_ds.append(d)\n",
    "    for d in subscriber_data:\n",
    "        subscriber_data_ds.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "9wZRIKrq2Kec",
    "outputId": "6a5e45aa-32cb-47da-d25c-325f6cfe106b",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train positive label: 5273 - train negative label: 64774\n",
      "up and down sampling => train positive label: 47457 - train negative label: 64774\n",
      "Test positive label: 580 - Test negative label: 7203\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.5617 - accuracy: 0.7180 - mae: 0.3770 - mse: 0.1882 - auc_11: 0.7807\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.4749 - accuracy: 0.7860 - mae: 0.3086 - mse: 0.1526 - auc_11: 0.8545\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.4400 - accuracy: 0.8043 - mae: 0.2826 - mse: 0.1399 - auc_11: 0.8766\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.4111 - accuracy: 0.8185 - mae: 0.2631 - mse: 0.1300 - auc_11: 0.8931\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.3911 - accuracy: 0.8297 - mae: 0.2489 - mse: 0.1229 - auc_11: 0.9036\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.3754 - accuracy: 0.8346 - mae: 0.2384 - mse: 0.1180 - auc_11: 0.9110\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.3572 - accuracy: 0.8435 - mae: 0.2262 - mse: 0.1122 - auc_11: 0.9191\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.3432 - accuracy: 0.8504 - mae: 0.2174 - mse: 0.1073 - auc_11: 0.9251\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.3334 - accuracy: 0.8560 - mae: 0.2099 - mse: 0.1039 - auc_11: 0.9291\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.3215 - accuracy: 0.8603 - mae: 0.2021 - mse: 0.1001 - auc_11: 0.9339\n",
      "85\n",
      "85\n",
      "84\n",
      "83\n",
      "82\n",
      "81\n",
      "80\n",
      "78\n",
      "71\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.3138 - accuracy: 0.8662 - mae: 0.1964 - mse: 0.0971 - auc_11: 0.9366\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.3010 - accuracy: 0.8715 - mae: 0.1883 - mse: 0.0931 - auc_11: 0.9411\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.2920 - accuracy: 0.8757 - mae: 0.1823 - mse: 0.0901 - auc_11: 0.9443\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2845 - accuracy: 0.8810 - mae: 0.1764 - mse: 0.0872 - auc_11: 0.9470\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.2782 - accuracy: 0.8826 - mae: 0.1722 - mse: 0.0855 - auc_11: 0.9489\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.2698 - accuracy: 0.8885 - mae: 0.1660 - mse: 0.0822 - auc_11: 0.9518\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.2653 - accuracy: 0.8898 - mae: 0.1632 - mse: 0.0807 - auc_11: 0.9536\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.2562 - accuracy: 0.8942 - mae: 0.1575 - mse: 0.0780 - auc_11: 0.9561\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.2506 - accuracy: 0.8971 - mae: 0.1535 - mse: 0.0760 - auc_11: 0.9578\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.2421 - accuracy: 0.9024 - mae: 0.1473 - mse: 0.0728 - auc_11: 0.9606\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "87\n",
      "86\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.2373 - accuracy: 0.9039 - mae: 0.1446 - mse: 0.0716 - auc_11: 0.9619\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2357 - accuracy: 0.9049 - mae: 0.1426 - mse: 0.0708 - auc_11: 0.9624\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2299 - accuracy: 0.9083 - mae: 0.1389 - mse: 0.0687 - auc_11: 0.9641\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2242 - accuracy: 0.9104 - mae: 0.1346 - mse: 0.0668 - auc_11: 0.9658\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2227 - accuracy: 0.9127 - mae: 0.1336 - mse: 0.0663 - auc_11: 0.9662\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2225 - accuracy: 0.9121 - mae: 0.1339 - mse: 0.0662 - auc_11: 0.9663\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2152 - accuracy: 0.9154 - mae: 0.1290 - mse: 0.0640 - auc_11: 0.9678\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2108 - accuracy: 0.9181 - mae: 0.1254 - mse: 0.0623 - auc_11: 0.9692\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2108 - accuracy: 0.9188 - mae: 0.1252 - mse: 0.0619 - auc_11: 0.9691\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2055 - accuracy: 0.9203 - mae: 0.1220 - mse: 0.0605 - auc_11: 0.9706\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "90\n",
      "90\n",
      "90\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2034 - accuracy: 0.9218 - mae: 0.1203 - mse: 0.0597 - auc_11: 0.9712\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2021 - accuracy: 0.9224 - mae: 0.1197 - mse: 0.0592 - auc_11: 0.9715\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1954 - accuracy: 0.9249 - mae: 0.1156 - mse: 0.0573 - auc_11: 0.9731\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 12ms/step - loss: 0.1967 - accuracy: 0.9257 - mae: 0.1157 - mse: 0.0572 - auc_11: 0.9730\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1922 - accuracy: 0.9270 - mae: 0.1130 - mse: 0.0560 - auc_11: 0.9740\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1889 - accuracy: 0.9286 - mae: 0.1104 - mse: 0.0547 - auc_11: 0.9748\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1889 - accuracy: 0.9281 - mae: 0.1107 - mse: 0.0548 - auc_11: 0.9750\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1860 - accuracy: 0.9297 - mae: 0.1087 - mse: 0.0539 - auc_11: 0.9754\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1849 - accuracy: 0.9307 - mae: 0.1077 - mse: 0.0533 - auc_11: 0.9757\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1863 - accuracy: 0.9293 - mae: 0.1088 - mse: 0.0541 - auc_11: 0.9756\n",
      "93\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1780 - accuracy: 0.9332 - mae: 0.1039 - mse: 0.0513 - auc_11: 0.9777\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1768 - accuracy: 0.9336 - mae: 0.1026 - mse: 0.0508 - auc_11: 0.9779\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1787 - accuracy: 0.9327 - mae: 0.1037 - mse: 0.0517 - auc_11: 0.9774\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1747 - accuracy: 0.9340 - mae: 0.1015 - mse: 0.0503 - auc_11: 0.9784\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1738 - accuracy: 0.9352 - mae: 0.1005 - mse: 0.0498 - auc_11: 0.9783\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1701 - accuracy: 0.9367 - mae: 0.0982 - mse: 0.0487 - auc_11: 0.9793\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1695 - accuracy: 0.9365 - mae: 0.0983 - mse: 0.0487 - auc_11: 0.9794\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1669 - accuracy: 0.9387 - mae: 0.0963 - mse: 0.0476 - auc_11: 0.9798\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1667 - accuracy: 0.9375 - mae: 0.0964 - mse: 0.0479 - auc_11: 0.9799\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1659 - accuracy: 0.9385 - mae: 0.0958 - mse: 0.0474 - auc_11: 0.9802\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "5\n",
      "244/244 [==============================] - 0s 2ms/step - loss: 0.3004 - accuracy: 0.9115 - mae: 0.1043 - mse: 0.0704 - auc_11: 0.8623\n",
      "train positive label: 5266 - train negative label: 64781\n",
      "up and down sampling => train positive label: 47394 - train negative label: 64781\n",
      "Test positive label: 587 - Test negative label: 7196\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.5897 - accuracy: 0.6804 - mae: 0.4017 - mse: 0.2009 - auc_12: 0.7478\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.4836 - accuracy: 0.7798 - mae: 0.3161 - mse: 0.1563 - auc_12: 0.8482\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.4415 - accuracy: 0.8020 - mae: 0.2848 - mse: 0.1405 - auc_12: 0.8757\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.4098 - accuracy: 0.8172 - mae: 0.2630 - mse: 0.1299 - auc_12: 0.8935\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.3864 - accuracy: 0.8298 - mae: 0.2467 - mse: 0.1220 - auc_12: 0.9052\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.3605 - accuracy: 0.8413 - mae: 0.2293 - mse: 0.1134 - auc_12: 0.9173\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.3471 - accuracy: 0.8488 - mae: 0.2196 - mse: 0.1085 - auc_12: 0.9230\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.3300 - accuracy: 0.8581 - mae: 0.2079 - mse: 0.1028 - auc_12: 0.9301\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.3138 - accuracy: 0.8652 - mae: 0.1973 - mse: 0.0976 - auc_12: 0.9359\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.3041 - accuracy: 0.8711 - mae: 0.1899 - mse: 0.0938 - auc_12: 0.9397\n",
      "86\n",
      "85\n",
      "84\n",
      "84\n",
      "82\n",
      "81\n",
      "80\n",
      "77\n",
      "68\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2925 - accuracy: 0.8774 - mae: 0.1823 - mse: 0.0901 - auc_12: 0.9434\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2856 - accuracy: 0.8813 - mae: 0.1769 - mse: 0.0876 - auc_12: 0.9458\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2773 - accuracy: 0.8849 - mae: 0.1716 - mse: 0.0848 - auc_12: 0.9488\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2682 - accuracy: 0.8894 - mae: 0.1648 - mse: 0.0816 - auc_12: 0.9518\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2619 - accuracy: 0.8933 - mae: 0.1603 - mse: 0.0793 - auc_12: 0.9538\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2567 - accuracy: 0.8957 - mae: 0.1566 - mse: 0.0777 - auc_12: 0.9552\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2519 - accuracy: 0.8987 - mae: 0.1538 - mse: 0.0760 - auc_12: 0.9566\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2451 - accuracy: 0.9009 - mae: 0.1495 - mse: 0.0741 - auc_12: 0.9590\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2438 - accuracy: 0.9032 - mae: 0.1472 - mse: 0.0729 - auc_12: 0.9597\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2408 - accuracy: 0.9045 - mae: 0.1453 - mse: 0.0720 - auc_12: 0.9600\n",
      "90\n",
      "90\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2321 - accuracy: 0.9077 - mae: 0.1399 - mse: 0.0693 - auc_12: 0.9627\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2323 - accuracy: 0.9082 - mae: 0.1401 - mse: 0.0694 - auc_12: 0.9630\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2265 - accuracy: 0.9101 - mae: 0.1361 - mse: 0.0675 - auc_12: 0.9646\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2228 - accuracy: 0.9129 - mae: 0.1331 - mse: 0.0661 - auc_12: 0.9659\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2190 - accuracy: 0.9151 - mae: 0.1308 - mse: 0.0647 - auc_12: 0.9670\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2140 - accuracy: 0.9159 - mae: 0.1275 - mse: 0.0634 - auc_12: 0.9683\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2164 - accuracy: 0.9159 - mae: 0.1285 - mse: 0.0639 - auc_12: 0.9678\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2100 - accuracy: 0.9187 - mae: 0.1245 - mse: 0.0617 - auc_12: 0.9692\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2070 - accuracy: 0.9196 - mae: 0.1226 - mse: 0.0611 - auc_12: 0.9703\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2006 - accuracy: 0.9226 - mae: 0.1190 - mse: 0.0589 - auc_12: 0.9717\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "90\n",
      "90\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.2010 - accuracy: 0.9217 - mae: 0.1191 - mse: 0.0592 - auc_12: 0.9718\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1988 - accuracy: 0.9238 - mae: 0.1175 - mse: 0.0582 - auc_12: 0.9723\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1973 - accuracy: 0.9245 - mae: 0.1161 - mse: 0.0575 - auc_12: 0.9727\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1922 - accuracy: 0.9271 - mae: 0.1122 - mse: 0.0560 - auc_12: 0.9739\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1886 - accuracy: 0.9275 - mae: 0.1110 - mse: 0.0552 - auc_12: 0.9747\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1881 - accuracy: 0.9288 - mae: 0.1100 - mse: 0.0547 - auc_12: 0.9747\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1899 - accuracy: 0.9284 - mae: 0.1106 - mse: 0.0548 - auc_12: 0.9745\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1852 - accuracy: 0.9304 - mae: 0.1078 - mse: 0.0535 - auc_12: 0.9754\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1850 - accuracy: 0.9302 - mae: 0.1078 - mse: 0.0538 - auc_12: 0.9755\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1777 - accuracy: 0.9335 - mae: 0.1037 - mse: 0.0513 - auc_12: 0.9775\n",
      "93\n",
      "93\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1785 - accuracy: 0.9329 - mae: 0.1037 - mse: 0.0515 - auc_12: 0.9773\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1742 - accuracy: 0.9354 - mae: 0.1010 - mse: 0.0500 - auc_12: 0.9783\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1742 - accuracy: 0.9344 - mae: 0.1006 - mse: 0.0501 - auc_12: 0.9784\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1760 - accuracy: 0.9341 - mae: 0.1015 - mse: 0.0504 - auc_12: 0.9778\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1733 - accuracy: 0.9358 - mae: 0.1006 - mse: 0.0495 - auc_12: 0.9785\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1680 - accuracy: 0.9383 - mae: 0.0964 - mse: 0.0480 - auc_12: 0.9796\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1654 - accuracy: 0.9384 - mae: 0.0953 - mse: 0.0474 - auc_12: 0.9799\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1690 - accuracy: 0.9374 - mae: 0.0975 - mse: 0.0484 - auc_12: 0.9795\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1687 - accuracy: 0.9374 - mae: 0.0973 - mse: 0.0483 - auc_12: 0.9796\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1622 - accuracy: 0.9404 - mae: 0.0930 - mse: 0.0463 - auc_12: 0.9806\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1617 - accuracy: 0.9403 - mae: 0.0929 - mse: 0.0461 - auc_12: 0.9807\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1603 - accuracy: 0.9413 - mae: 0.0917 - mse: 0.0455 - auc_12: 0.9810\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1630 - accuracy: 0.9404 - mae: 0.0935 - mse: 0.0464 - auc_12: 0.9806\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1603 - accuracy: 0.9414 - mae: 0.0917 - mse: 0.0457 - auc_12: 0.9808\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1584 - accuracy: 0.9423 - mae: 0.0910 - mse: 0.0448 - auc_12: 0.9819\n",
      "Epoch 6/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1649 - accuracy: 0.9394 - mae: 0.0946 - mse: 0.0470 - auc_12: 0.9804\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1631 - accuracy: 0.9411 - mae: 0.0927 - mse: 0.0459 - auc_12: 0.9808\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1547 - accuracy: 0.9430 - mae: 0.0888 - mse: 0.0440 - auc_12: 0.9827\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1524 - accuracy: 0.9445 - mae: 0.0870 - mse: 0.0431 - auc_12: 0.9828\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1553 - accuracy: 0.9439 - mae: 0.0883 - mse: 0.0437 - auc_12: 0.9823\n",
      "94\n",
      "94\n",
      "94\n",
      "93\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "Epoch 1/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1510 - accuracy: 0.9450 - mae: 0.0860 - mse: 0.0427 - auc_12: 0.9831\n",
      "Epoch 2/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1498 - accuracy: 0.9458 - mae: 0.0843 - mse: 0.0422 - auc_12: 0.9835\n",
      "Epoch 3/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1540 - accuracy: 0.9440 - mae: 0.0880 - mse: 0.0435 - auc_12: 0.9826\n",
      "Epoch 4/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1508 - accuracy: 0.9452 - mae: 0.0857 - mse: 0.0425 - auc_12: 0.9832\n",
      "Epoch 5/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1492 - accuracy: 0.9455 - mae: 0.0855 - mse: 0.0422 - auc_12: 0.9838\n",
      "Epoch 6/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1489 - accuracy: 0.9466 - mae: 0.0843 - mse: 0.0417 - auc_12: 0.9836\n",
      "Epoch 7/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1466 - accuracy: 0.9468 - mae: 0.0832 - mse: 0.0415 - auc_12: 0.9841\n",
      "Epoch 8/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1463 - accuracy: 0.9482 - mae: 0.0823 - mse: 0.0406 - auc_12: 0.9840\n",
      "Epoch 9/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1509 - accuracy: 0.9462 - mae: 0.0856 - mse: 0.0423 - auc_12: 0.9833\n",
      "Epoch 10/10\n",
      "877/877 [==============================] - 10s 11ms/step - loss: 0.1445 - accuracy: 0.9475 - mae: 0.0817 - mse: 0.0408 - auc_12: 0.9844\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "7\n",
      "244/244 [==============================] - 0s 2ms/step - loss: 0.2444 - accuracy: 0.9188 - mae: 0.1100 - mse: 0.0657 - auc_12: 0.8747\n",
      "train positive label: 5291 - train negative label: 64756\n",
      "up and down sampling => train positive label: 47619 - train negative label: 64756\n",
      "Test positive label: 562 - Test negative label: 7221\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.5641 - accuracy: 0.7185 - mae: 0.3779 - mse: 0.1887 - auc_13: 0.7806\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.4783 - accuracy: 0.7843 - mae: 0.3110 - mse: 0.1536 - auc_13: 0.8527\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.4406 - accuracy: 0.8039 - mae: 0.2831 - mse: 0.1399 - auc_13: 0.8769\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.4110 - accuracy: 0.8168 - mae: 0.2629 - mse: 0.1299 - auc_13: 0.8938\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.3937 - accuracy: 0.8271 - mae: 0.2512 - mse: 0.1242 - auc_13: 0.9023\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.3709 - accuracy: 0.8376 - mae: 0.2358 - mse: 0.1166 - auc_13: 0.9133\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.3570 - accuracy: 0.8435 - mae: 0.2266 - mse: 0.1122 - auc_13: 0.9192\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.3412 - accuracy: 0.8513 - mae: 0.2162 - mse: 0.1068 - auc_13: 0.9261\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.3296 - accuracy: 0.8568 - mae: 0.2076 - mse: 0.1028 - auc_13: 0.9307\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.3199 - accuracy: 0.8624 - mae: 0.2008 - mse: 0.0993 - auc_13: 0.9346\n",
      "85\n",
      "85\n",
      "84\n",
      "83\n",
      "82\n",
      "81\n",
      "80\n",
      "78\n",
      "71\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.3068 - accuracy: 0.8677 - mae: 0.1928 - mse: 0.0953 - auc_13: 0.9393\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2983 - accuracy: 0.8721 - mae: 0.1869 - mse: 0.0924 - auc_13: 0.9424\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2948 - accuracy: 0.8739 - mae: 0.1839 - mse: 0.0909 - auc_13: 0.9439\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2848 - accuracy: 0.8789 - mae: 0.1773 - mse: 0.0875 - auc_13: 0.9475\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2735 - accuracy: 0.8843 - mae: 0.1699 - mse: 0.0839 - auc_13: 0.9511\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2701 - accuracy: 0.8869 - mae: 0.1670 - mse: 0.0823 - auc_13: 0.9526\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2632 - accuracy: 0.8907 - mae: 0.1619 - mse: 0.0798 - auc_13: 0.9546\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2560 - accuracy: 0.8930 - mae: 0.1575 - mse: 0.0778 - auc_13: 0.9568\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2533 - accuracy: 0.8961 - mae: 0.1550 - mse: 0.0766 - auc_13: 0.9577\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2439 - accuracy: 0.8995 - mae: 0.1495 - mse: 0.0736 - auc_13: 0.9606\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "87\n",
      "87\n",
      "87\n",
      "86\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2395 - accuracy: 0.9023 - mae: 0.1457 - mse: 0.0721 - auc_13: 0.9618\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2332 - accuracy: 0.9058 - mae: 0.1416 - mse: 0.0699 - auc_13: 0.9638\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2324 - accuracy: 0.9071 - mae: 0.1405 - mse: 0.0694 - auc_13: 0.9641\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2285 - accuracy: 0.9084 - mae: 0.1379 - mse: 0.0681 - auc_13: 0.9651\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2205 - accuracy: 0.9112 - mae: 0.1334 - mse: 0.0657 - auc_13: 0.9672\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2158 - accuracy: 0.9143 - mae: 0.1294 - mse: 0.0642 - auc_13: 0.9685\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2171 - accuracy: 0.9141 - mae: 0.1299 - mse: 0.0642 - auc_13: 0.9683\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2124 - accuracy: 0.9154 - mae: 0.1274 - mse: 0.0631 - auc_13: 0.9696\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2107 - accuracy: 0.9169 - mae: 0.1257 - mse: 0.0620 - auc_13: 0.9703\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2053 - accuracy: 0.9192 - mae: 0.1224 - mse: 0.0604 - auc_13: 0.9713\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.2008 - accuracy: 0.9212 - mae: 0.1192 - mse: 0.0591 - auc_13: 0.9726\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1954 - accuracy: 0.9236 - mae: 0.1159 - mse: 0.0573 - auc_13: 0.9738\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1964 - accuracy: 0.9227 - mae: 0.1168 - mse: 0.0576 - auc_13: 0.9738\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1875 - accuracy: 0.9263 - mae: 0.1110 - mse: 0.0550 - auc_13: 0.9758\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1871 - accuracy: 0.9278 - mae: 0.1100 - mse: 0.0545 - auc_13: 0.9757\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1848 - accuracy: 0.9282 - mae: 0.1086 - mse: 0.0538 - auc_13: 0.9764\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1819 - accuracy: 0.9308 - mae: 0.1069 - mse: 0.0525 - auc_13: 0.9770\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1771 - accuracy: 0.9328 - mae: 0.1035 - mse: 0.0513 - auc_13: 0.9782\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1785 - accuracy: 0.9310 - mae: 0.1044 - mse: 0.0517 - auc_13: 0.9780\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1733 - accuracy: 0.9342 - mae: 0.1011 - mse: 0.0500 - auc_13: 0.9790\n",
      "93\n",
      "93\n",
      "93\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1758 - accuracy: 0.9332 - mae: 0.1020 - mse: 0.0506 - auc_13: 0.9786\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1709 - accuracy: 0.9350 - mae: 0.0995 - mse: 0.0490 - auc_13: 0.9797\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1668 - accuracy: 0.9363 - mae: 0.0974 - mse: 0.0479 - auc_13: 0.9807\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1683 - accuracy: 0.9368 - mae: 0.0971 - mse: 0.0483 - auc_13: 0.9801\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1649 - accuracy: 0.9380 - mae: 0.0955 - mse: 0.0473 - auc_13: 0.9808\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1615 - accuracy: 0.9396 - mae: 0.0935 - mse: 0.0462 - auc_13: 0.9817\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1627 - accuracy: 0.9388 - mae: 0.0940 - mse: 0.0467 - auc_13: 0.9813\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1636 - accuracy: 0.9394 - mae: 0.0942 - mse: 0.0468 - auc_13: 0.9811\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1591 - accuracy: 0.9414 - mae: 0.0913 - mse: 0.0451 - auc_13: 0.9820\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1588 - accuracy: 0.9420 - mae: 0.0908 - mse: 0.0448 - auc_13: 0.9823\n",
      "94\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1548 - accuracy: 0.9428 - mae: 0.0890 - mse: 0.0439 - auc_13: 0.9831\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1529 - accuracy: 0.9433 - mae: 0.0878 - mse: 0.0434 - auc_13: 0.9834\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1532 - accuracy: 0.9433 - mae: 0.0878 - mse: 0.0434 - auc_13: 0.9834\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1500 - accuracy: 0.9455 - mae: 0.0854 - mse: 0.0423 - auc_13: 0.9838\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1516 - accuracy: 0.9449 - mae: 0.0865 - mse: 0.0426 - auc_13: 0.9836\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1502 - accuracy: 0.9450 - mae: 0.0856 - mse: 0.0423 - auc_13: 0.9841\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1476 - accuracy: 0.9461 - mae: 0.0838 - mse: 0.0416 - auc_13: 0.9844\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1472 - accuracy: 0.9462 - mae: 0.0839 - mse: 0.0414 - auc_13: 0.9844\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1523 - accuracy: 0.9442 - mae: 0.0863 - mse: 0.0428 - auc_13: 0.9836\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1437 - accuracy: 0.9468 - mae: 0.0817 - mse: 0.0407 - auc_13: 0.9851\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "6\n",
      "244/244 [==============================] - 0s 2ms/step - loss: 0.2794 - accuracy: 0.9103 - mae: 0.0988 - mse: 0.0657 - auc_13: 0.8712\n",
      "train positive label: 5258 - train negative label: 64789\n",
      "up and down sampling => train positive label: 47322 - train negative label: 64789\n",
      "Test positive label: 595 - Test negative label: 7188\n",
      "Epoch 1/10\n",
      "876/876 [==============================] - 9s 10ms/step - loss: 0.5691 - accuracy: 0.7098 - mae: 0.3824 - mse: 0.1914 - auc_14: 0.7720\n",
      "Epoch 2/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.4824 - accuracy: 0.7808 - mae: 0.3143 - mse: 0.1556 - auc_14: 0.8492\n",
      "Epoch 3/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.4462 - accuracy: 0.8005 - mae: 0.2883 - mse: 0.1425 - auc_14: 0.8728\n",
      "Epoch 4/10\n",
      "876/876 [==============================] - 9s 10ms/step - loss: 0.4232 - accuracy: 0.8107 - mae: 0.2720 - mse: 0.1343 - auc_14: 0.8867\n",
      "Epoch 5/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.3938 - accuracy: 0.8259 - mae: 0.2521 - mse: 0.1244 - auc_14: 0.9025\n",
      "Epoch 6/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.3744 - accuracy: 0.8350 - mae: 0.2388 - mse: 0.1177 - auc_14: 0.9119\n",
      "Epoch 7/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.3554 - accuracy: 0.8433 - mae: 0.2259 - mse: 0.1118 - auc_14: 0.9201\n",
      "Epoch 8/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.3405 - accuracy: 0.8509 - mae: 0.2158 - mse: 0.1066 - auc_14: 0.9265\n",
      "Epoch 9/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.3237 - accuracy: 0.8597 - mae: 0.2035 - mse: 0.1008 - auc_14: 0.9332\n",
      "Epoch 10/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.3173 - accuracy: 0.8637 - mae: 0.1991 - mse: 0.0984 - auc_14: 0.9362\n",
      "85\n",
      "85\n",
      "84\n",
      "83\n",
      "82\n",
      "81\n",
      "80\n",
      "78\n",
      "70\n",
      "Epoch 1/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.3054 - accuracy: 0.8683 - mae: 0.1912 - mse: 0.0946 - auc_14: 0.9405\n",
      "Epoch 2/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2941 - accuracy: 0.8747 - mae: 0.1831 - mse: 0.0905 - auc_14: 0.9446\n",
      "Epoch 3/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2859 - accuracy: 0.8799 - mae: 0.1773 - mse: 0.0877 - auc_14: 0.9472\n",
      "Epoch 4/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2773 - accuracy: 0.8834 - mae: 0.1713 - mse: 0.0851 - auc_14: 0.9500\n",
      "Epoch 5/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2724 - accuracy: 0.8858 - mae: 0.1678 - mse: 0.0832 - auc_14: 0.9519\n",
      "Epoch 6/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2632 - accuracy: 0.8912 - mae: 0.1612 - mse: 0.0800 - auc_14: 0.9548\n",
      "Epoch 7/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2592 - accuracy: 0.8926 - mae: 0.1583 - mse: 0.0787 - auc_14: 0.9559\n",
      "Epoch 8/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2556 - accuracy: 0.8955 - mae: 0.1558 - mse: 0.0773 - auc_14: 0.9572\n",
      "Epoch 9/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2506 - accuracy: 0.8967 - mae: 0.1525 - mse: 0.0758 - auc_14: 0.9585\n",
      "Epoch 10/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2445 - accuracy: 0.9005 - mae: 0.1480 - mse: 0.0735 - auc_14: 0.9605\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "87\n",
      "87\n",
      "86\n",
      "Epoch 1/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2391 - accuracy: 0.9034 - mae: 0.1444 - mse: 0.0717 - auc_14: 0.9621\n",
      "Epoch 2/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2338 - accuracy: 0.9058 - mae: 0.1409 - mse: 0.0702 - auc_14: 0.9637\n",
      "Epoch 3/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2335 - accuracy: 0.9057 - mae: 0.1406 - mse: 0.0696 - auc_14: 0.9638\n",
      "Epoch 4/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2238 - accuracy: 0.9117 - mae: 0.1339 - mse: 0.0663 - auc_14: 0.9665\n",
      "Epoch 5/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2299 - accuracy: 0.9085 - mae: 0.1369 - mse: 0.0682 - auc_14: 0.9649\n",
      "Epoch 6/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2181 - accuracy: 0.9139 - mae: 0.1300 - mse: 0.0647 - auc_14: 0.9679\n",
      "Epoch 7/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2153 - accuracy: 0.9159 - mae: 0.1278 - mse: 0.0634 - auc_14: 0.9689\n",
      "Epoch 8/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2170 - accuracy: 0.9149 - mae: 0.1283 - mse: 0.0640 - auc_14: 0.9684\n",
      "Epoch 9/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2148 - accuracy: 0.9157 - mae: 0.1270 - mse: 0.0632 - auc_14: 0.9689\n",
      "Epoch 10/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2132 - accuracy: 0.9159 - mae: 0.1264 - mse: 0.0628 - auc_14: 0.9693\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "90\n",
      "91\n",
      "90\n",
      "90\n",
      "90\n",
      "Epoch 1/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2048 - accuracy: 0.9199 - mae: 0.1212 - mse: 0.0603 - auc_14: 0.9712\n",
      "Epoch 2/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2036 - accuracy: 0.9198 - mae: 0.1202 - mse: 0.0598 - auc_14: 0.9718\n",
      "Epoch 3/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2065 - accuracy: 0.9198 - mae: 0.1213 - mse: 0.0605 - auc_14: 0.9711\n",
      "Epoch 4/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.2001 - accuracy: 0.9225 - mae: 0.1174 - mse: 0.0585 - auc_14: 0.9727\n",
      "Epoch 5/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1986 - accuracy: 0.9229 - mae: 0.1164 - mse: 0.0578 - auc_14: 0.9731\n",
      "Epoch 6/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1924 - accuracy: 0.9251 - mae: 0.1128 - mse: 0.0564 - auc_14: 0.9745\n",
      "Epoch 7/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1917 - accuracy: 0.9256 - mae: 0.1127 - mse: 0.0558 - auc_14: 0.9748\n",
      "Epoch 8/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1923 - accuracy: 0.9255 - mae: 0.1125 - mse: 0.0560 - auc_14: 0.9743\n",
      "Epoch 9/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1906 - accuracy: 0.9273 - mae: 0.1108 - mse: 0.0553 - auc_14: 0.9751\n",
      "Epoch 10/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1883 - accuracy: 0.9278 - mae: 0.1097 - mse: 0.0546 - auc_14: 0.9754\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "91\n",
      "91\n",
      "91\n",
      "Epoch 1/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1885 - accuracy: 0.9276 - mae: 0.1099 - mse: 0.0547 - auc_14: 0.9755\n",
      "Epoch 2/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1900 - accuracy: 0.9272 - mae: 0.1104 - mse: 0.0549 - auc_14: 0.9752\n",
      "Epoch 3/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1851 - accuracy: 0.9293 - mae: 0.1077 - mse: 0.0538 - auc_14: 0.9762\n",
      "Epoch 4/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1835 - accuracy: 0.9300 - mae: 0.1067 - mse: 0.0532 - auc_14: 0.9763\n",
      "Epoch 5/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1815 - accuracy: 0.9312 - mae: 0.1056 - mse: 0.0524 - auc_14: 0.9771\n",
      "Epoch 6/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1774 - accuracy: 0.9327 - mae: 0.1028 - mse: 0.0512 - auc_14: 0.9781\n",
      "Epoch 7/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1827 - accuracy: 0.9311 - mae: 0.1055 - mse: 0.0526 - auc_14: 0.9770\n",
      "Epoch 8/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1804 - accuracy: 0.9321 - mae: 0.1046 - mse: 0.0519 - auc_14: 0.9778\n",
      "Epoch 9/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1799 - accuracy: 0.9308 - mae: 0.1048 - mse: 0.0521 - auc_14: 0.9775\n",
      "Epoch 10/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1743 - accuracy: 0.9336 - mae: 0.1008 - mse: 0.0504 - auc_14: 0.9789\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "Epoch 1/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1721 - accuracy: 0.9344 - mae: 0.0998 - mse: 0.0496 - auc_14: 0.9791\n",
      "Epoch 2/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1697 - accuracy: 0.9358 - mae: 0.0976 - mse: 0.0486 - auc_14: 0.9796\n",
      "Epoch 3/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1714 - accuracy: 0.9352 - mae: 0.0986 - mse: 0.0493 - auc_14: 0.9796\n",
      "Epoch 4/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1748 - accuracy: 0.9342 - mae: 0.1005 - mse: 0.0500 - auc_14: 0.9788\n",
      "Epoch 5/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1697 - accuracy: 0.9354 - mae: 0.0980 - mse: 0.0487 - auc_14: 0.9797\n",
      "Epoch 6/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1689 - accuracy: 0.9377 - mae: 0.0969 - mse: 0.0480 - auc_14: 0.9799\n",
      "Epoch 7/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1691 - accuracy: 0.9362 - mae: 0.0973 - mse: 0.0485 - auc_14: 0.9798\n",
      "Epoch 8/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1649 - accuracy: 0.9385 - mae: 0.0944 - mse: 0.0471 - auc_14: 0.9809\n",
      "Epoch 9/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1642 - accuracy: 0.9383 - mae: 0.0941 - mse: 0.0469 - auc_14: 0.9810\n",
      "Epoch 10/10\n",
      "876/876 [==============================] - 9s 11ms/step - loss: 0.1647 - accuracy: 0.9383 - mae: 0.0947 - mse: 0.0469 - auc_14: 0.9807\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "6\n",
      "244/244 [==============================] - 0s 2ms/step - loss: 0.2966 - accuracy: 0.9147 - mae: 0.0989 - mse: 0.0663 - auc_14: 0.8761\n",
      "train positive label: 5281 - train negative label: 64766\n",
      "up and down sampling => train positive label: 47529 - train negative label: 64766\n",
      "Test positive label: 572 - Test negative label: 7211\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.5773 - accuracy: 0.7059 - mae: 0.3891 - mse: 0.1946 - auc_15: 0.7650\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.4794 - accuracy: 0.7831 - mae: 0.3118 - mse: 0.1542 - auc_15: 0.8514\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.4400 - accuracy: 0.8045 - mae: 0.2836 - mse: 0.1401 - auc_15: 0.8766\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.4073 - accuracy: 0.8189 - mae: 0.2608 - mse: 0.1289 - auc_15: 0.8951\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3855 - accuracy: 0.8298 - mae: 0.2458 - mse: 0.1214 - auc_15: 0.9064\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3640 - accuracy: 0.8410 - mae: 0.2313 - mse: 0.1141 - auc_15: 0.9164\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3479 - accuracy: 0.8490 - mae: 0.2200 - mse: 0.1088 - auc_15: 0.9233\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3350 - accuracy: 0.8559 - mae: 0.2106 - mse: 0.1041 - auc_15: 0.9287\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3237 - accuracy: 0.8608 - mae: 0.2033 - mse: 0.1005 - auc_15: 0.9328\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3123 - accuracy: 0.8669 - mae: 0.1959 - mse: 0.0966 - auc_15: 0.9373\n",
      "86\n",
      "85\n",
      "84\n",
      "84\n",
      "82\n",
      "81\n",
      "80\n",
      "78\n",
      "70\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.3014 - accuracy: 0.8719 - mae: 0.1887 - mse: 0.0932 - auc_15: 0.9410\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2953 - accuracy: 0.8758 - mae: 0.1837 - mse: 0.0908 - auc_15: 0.9433\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2833 - accuracy: 0.8810 - mae: 0.1758 - mse: 0.0868 - auc_15: 0.9475\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2798 - accuracy: 0.8837 - mae: 0.1727 - mse: 0.0855 - auc_15: 0.9487\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2722 - accuracy: 0.8879 - mae: 0.1677 - mse: 0.0828 - auc_15: 0.9514\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2656 - accuracy: 0.8903 - mae: 0.1630 - mse: 0.0808 - auc_15: 0.9535\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2595 - accuracy: 0.8944 - mae: 0.1586 - mse: 0.0784 - auc_15: 0.9555\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2541 - accuracy: 0.8953 - mae: 0.1557 - mse: 0.0769 - auc_15: 0.9570\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2509 - accuracy: 0.8973 - mae: 0.1525 - mse: 0.0756 - auc_15: 0.9583\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2443 - accuracy: 0.9010 - mae: 0.1479 - mse: 0.0733 - auc_15: 0.9600\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "87\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2416 - accuracy: 0.9015 - mae: 0.1467 - mse: 0.0726 - auc_15: 0.9610\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2361 - accuracy: 0.9049 - mae: 0.1425 - mse: 0.0704 - auc_15: 0.9625\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2330 - accuracy: 0.9067 - mae: 0.1404 - mse: 0.0696 - auc_15: 0.9633\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2300 - accuracy: 0.9089 - mae: 0.1378 - mse: 0.0684 - auc_15: 0.9643\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2235 - accuracy: 0.9109 - mae: 0.1344 - mse: 0.0665 - auc_15: 0.9662\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2177 - accuracy: 0.9141 - mae: 0.1301 - mse: 0.0643 - auc_15: 0.9679\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2170 - accuracy: 0.9149 - mae: 0.1294 - mse: 0.0641 - auc_15: 0.9677\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2123 - accuracy: 0.9154 - mae: 0.1269 - mse: 0.0629 - auc_15: 0.9693\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2127 - accuracy: 0.9160 - mae: 0.1267 - mse: 0.0628 - auc_15: 0.9692\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2100 - accuracy: 0.9171 - mae: 0.1250 - mse: 0.0620 - auc_15: 0.9700\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2045 - accuracy: 0.9202 - mae: 0.1212 - mse: 0.0600 - auc_15: 0.9712\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2013 - accuracy: 0.9210 - mae: 0.1193 - mse: 0.0592 - auc_15: 0.9720\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1988 - accuracy: 0.9233 - mae: 0.1171 - mse: 0.0581 - auc_15: 0.9726\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1979 - accuracy: 0.9235 - mae: 0.1172 - mse: 0.0579 - auc_15: 0.9729\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1950 - accuracy: 0.9242 - mae: 0.1149 - mse: 0.0570 - auc_15: 0.9738\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1932 - accuracy: 0.9248 - mae: 0.1139 - mse: 0.0564 - auc_15: 0.9741\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1900 - accuracy: 0.9260 - mae: 0.1119 - mse: 0.0555 - auc_15: 0.9749\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1853 - accuracy: 0.9288 - mae: 0.1084 - mse: 0.0537 - auc_15: 0.9763\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1868 - accuracy: 0.9281 - mae: 0.1093 - mse: 0.0543 - auc_15: 0.9757\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1811 - accuracy: 0.9310 - mae: 0.1058 - mse: 0.0525 - auc_15: 0.9770\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1805 - accuracy: 0.9320 - mae: 0.1052 - mse: 0.0521 - auc_15: 0.9772\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1839 - accuracy: 0.9296 - mae: 0.1078 - mse: 0.0532 - auc_15: 0.9766\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1789 - accuracy: 0.9320 - mae: 0.1040 - mse: 0.0516 - auc_15: 0.9777\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1783 - accuracy: 0.9324 - mae: 0.1036 - mse: 0.0516 - auc_15: 0.9779\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1802 - accuracy: 0.9311 - mae: 0.1049 - mse: 0.0521 - auc_15: 0.9777\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1725 - accuracy: 0.9344 - mae: 0.1004 - mse: 0.0498 - auc_15: 0.9790\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1698 - accuracy: 0.9355 - mae: 0.0985 - mse: 0.0489 - auc_15: 0.9797\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1653 - accuracy: 0.9375 - mae: 0.0958 - mse: 0.0475 - auc_15: 0.9804\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1691 - accuracy: 0.9363 - mae: 0.0980 - mse: 0.0485 - auc_15: 0.9797\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1656 - accuracy: 0.9383 - mae: 0.0955 - mse: 0.0472 - auc_15: 0.9805\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "92\n",
      "93\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1633 - accuracy: 0.9396 - mae: 0.0943 - mse: 0.0466 - auc_15: 0.9809\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1654 - accuracy: 0.9381 - mae: 0.0949 - mse: 0.0472 - auc_15: 0.9804\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1617 - accuracy: 0.9390 - mae: 0.0933 - mse: 0.0463 - auc_15: 0.9814\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1639 - accuracy: 0.9394 - mae: 0.0941 - mse: 0.0467 - auc_15: 0.9810\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1586 - accuracy: 0.9415 - mae: 0.0910 - mse: 0.0450 - auc_15: 0.9821\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1632 - accuracy: 0.9393 - mae: 0.0936 - mse: 0.0464 - auc_15: 0.9810\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1548 - accuracy: 0.9425 - mae: 0.0886 - mse: 0.0440 - auc_15: 0.9825\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1538 - accuracy: 0.9433 - mae: 0.0882 - mse: 0.0435 - auc_15: 0.9828\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1554 - accuracy: 0.9422 - mae: 0.0884 - mse: 0.0439 - auc_15: 0.9825\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1491 - accuracy: 0.9453 - mae: 0.0853 - mse: 0.0422 - auc_15: 0.9838\n",
      "94\n",
      "94\n",
      "94\n",
      "93\n",
      "94\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1501 - accuracy: 0.9448 - mae: 0.0854 - mse: 0.0425 - auc_15: 0.9835\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1544 - accuracy: 0.9436 - mae: 0.0884 - mse: 0.0435 - auc_15: 0.9830\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1496 - accuracy: 0.9448 - mae: 0.0856 - mse: 0.0423 - auc_15: 0.9840\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1442 - accuracy: 0.9463 - mae: 0.0824 - mse: 0.0409 - auc_15: 0.9848\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1465 - accuracy: 0.9462 - mae: 0.0829 - mse: 0.0412 - auc_15: 0.9842\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1439 - accuracy: 0.9482 - mae: 0.0815 - mse: 0.0403 - auc_15: 0.9849\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1480 - accuracy: 0.9454 - mae: 0.0843 - mse: 0.0420 - auc_15: 0.9842\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1455 - accuracy: 0.9465 - mae: 0.0824 - mse: 0.0410 - auc_15: 0.9846\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1462 - accuracy: 0.9465 - mae: 0.0832 - mse: 0.0411 - auc_15: 0.9847\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 11ms/step - loss: 0.1452 - accuracy: 0.9476 - mae: 0.0820 - mse: 0.0405 - auc_15: 0.9848\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "7\n",
      "244/244 [==============================] - 0s 2ms/step - loss: 0.2842 - accuracy: 0.9211 - mae: 0.0888 - mse: 0.0593 - auc_15: 0.8739\n",
      "train positive label: 5285 - train negative label: 64762\n",
      "up and down sampling => train positive label: 47565 - train negative label: 64762\n",
      "Test positive label: 568 - Test negative label: 7215\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.5668 - accuracy: 0.7160 - mae: 0.3799 - mse: 0.1900 - auc_16: 0.7770\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.4838 - accuracy: 0.7840 - mae: 0.3146 - mse: 0.1554 - auc_16: 0.8487\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.4482 - accuracy: 0.8021 - mae: 0.2884 - mse: 0.1422 - auc_16: 0.8722\n",
      "Epoch 4/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "878/878 [==============================] - 9s 10ms/step - loss: 0.4208 - accuracy: 0.8154 - mae: 0.2691 - mse: 0.1329 - auc_16: 0.8878\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3965 - accuracy: 0.8249 - mae: 0.2531 - mse: 0.1252 - auc_16: 0.9003\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3747 - accuracy: 0.8348 - mae: 0.2382 - mse: 0.1179 - auc_16: 0.9110\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3572 - accuracy: 0.8428 - mae: 0.2272 - mse: 0.1124 - auc_16: 0.9187\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3426 - accuracy: 0.8501 - mae: 0.2164 - mse: 0.1073 - auc_16: 0.9248\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3317 - accuracy: 0.8560 - mae: 0.2088 - mse: 0.1032 - auc_16: 0.9298\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3218 - accuracy: 0.8616 - mae: 0.2016 - mse: 0.0995 - auc_16: 0.9336\n",
      "85\n",
      "85\n",
      "84\n",
      "83\n",
      "82\n",
      "81\n",
      "80\n",
      "78\n",
      "71\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3091 - accuracy: 0.8678 - mae: 0.1929 - mse: 0.0957 - auc_16: 0.9381\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.3011 - accuracy: 0.8736 - mae: 0.1869 - mse: 0.0928 - auc_16: 0.9412\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2952 - accuracy: 0.8754 - mae: 0.1827 - mse: 0.0905 - auc_16: 0.9435\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2868 - accuracy: 0.8800 - mae: 0.1766 - mse: 0.0876 - auc_16: 0.9464\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2814 - accuracy: 0.8832 - mae: 0.1733 - mse: 0.0861 - auc_16: 0.9478\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2763 - accuracy: 0.8852 - mae: 0.1696 - mse: 0.0842 - auc_16: 0.9498\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2671 - accuracy: 0.8911 - mae: 0.1629 - mse: 0.0809 - auc_16: 0.9526\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2623 - accuracy: 0.8930 - mae: 0.1598 - mse: 0.0794 - auc_16: 0.9545\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2620 - accuracy: 0.8930 - mae: 0.1596 - mse: 0.0795 - auc_16: 0.9543\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2571 - accuracy: 0.8952 - mae: 0.1564 - mse: 0.0777 - auc_16: 0.9563\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "87\n",
      "87\n",
      "87\n",
      "86\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2506 - accuracy: 0.8990 - mae: 0.1517 - mse: 0.0751 - auc_16: 0.9582\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2421 - accuracy: 0.9016 - mae: 0.1466 - mse: 0.0728 - auc_16: 0.9610\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2421 - accuracy: 0.9031 - mae: 0.1461 - mse: 0.0726 - auc_16: 0.9608\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2398 - accuracy: 0.9037 - mae: 0.1441 - mse: 0.0716 - auc_16: 0.9618\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2325 - accuracy: 0.9077 - mae: 0.1392 - mse: 0.0692 - auc_16: 0.9637\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2285 - accuracy: 0.9099 - mae: 0.1373 - mse: 0.0679 - auc_16: 0.9649\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2307 - accuracy: 0.9090 - mae: 0.1377 - mse: 0.0685 - auc_16: 0.9641\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2214 - accuracy: 0.9136 - mae: 0.1315 - mse: 0.0654 - auc_16: 0.9669\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2164 - accuracy: 0.9144 - mae: 0.1291 - mse: 0.0641 - auc_16: 0.9678\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2176 - accuracy: 0.9140 - mae: 0.1295 - mse: 0.0645 - auc_16: 0.9677\n",
      "91\n",
      "91\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "89\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2187 - accuracy: 0.9144 - mae: 0.1302 - mse: 0.0647 - auc_16: 0.9675\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2103 - accuracy: 0.9184 - mae: 0.1241 - mse: 0.0619 - auc_16: 0.9697\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.2163 - accuracy: 0.9156 - mae: 0.1278 - mse: 0.0637 - auc_16: 0.9682\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2080 - accuracy: 0.9191 - mae: 0.1228 - mse: 0.0609 - auc_16: 0.9704\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2019 - accuracy: 0.9208 - mae: 0.1191 - mse: 0.0592 - auc_16: 0.9718\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1989 - accuracy: 0.9233 - mae: 0.1166 - mse: 0.0582 - auc_16: 0.9724\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1974 - accuracy: 0.9235 - mae: 0.1159 - mse: 0.0575 - auc_16: 0.9731\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.2009 - accuracy: 0.9224 - mae: 0.1179 - mse: 0.0587 - auc_16: 0.9724\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1986 - accuracy: 0.9236 - mae: 0.1166 - mse: 0.0580 - auc_16: 0.9729\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1906 - accuracy: 0.9274 - mae: 0.1115 - mse: 0.0554 - auc_16: 0.9748\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1887 - accuracy: 0.9280 - mae: 0.1102 - mse: 0.0548 - auc_16: 0.9752\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1859 - accuracy: 0.9290 - mae: 0.1086 - mse: 0.0540 - auc_16: 0.9760\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1843 - accuracy: 0.9297 - mae: 0.1072 - mse: 0.0533 - auc_16: 0.9763\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1881 - accuracy: 0.9280 - mae: 0.1098 - mse: 0.0544 - auc_16: 0.9758\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1850 - accuracy: 0.9294 - mae: 0.1079 - mse: 0.0535 - auc_16: 0.9761\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1836 - accuracy: 0.9310 - mae: 0.1064 - mse: 0.0529 - auc_16: 0.9762\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1800 - accuracy: 0.9324 - mae: 0.1047 - mse: 0.0518 - auc_16: 0.9772\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1810 - accuracy: 0.9318 - mae: 0.1049 - mse: 0.0521 - auc_16: 0.9771\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1784 - accuracy: 0.9323 - mae: 0.1037 - mse: 0.0515 - auc_16: 0.9778\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1762 - accuracy: 0.9340 - mae: 0.1019 - mse: 0.0505 - auc_16: 0.9782\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1737 - accuracy: 0.9345 - mae: 0.1007 - mse: 0.0500 - auc_16: 0.9788\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1753 - accuracy: 0.9337 - mae: 0.1015 - mse: 0.0505 - auc_16: 0.9782\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1682 - accuracy: 0.9369 - mae: 0.0972 - mse: 0.0482 - auc_16: 0.9798\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1709 - accuracy: 0.9365 - mae: 0.0987 - mse: 0.0488 - auc_16: 0.9794\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1629 - accuracy: 0.9388 - mae: 0.0939 - mse: 0.0468 - auc_16: 0.9810\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1679 - accuracy: 0.9368 - mae: 0.0969 - mse: 0.0482 - auc_16: 0.9802\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1664 - accuracy: 0.9380 - mae: 0.0958 - mse: 0.0474 - auc_16: 0.9806\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1657 - accuracy: 0.9387 - mae: 0.0950 - mse: 0.0472 - auc_16: 0.9806\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1605 - accuracy: 0.9408 - mae: 0.0918 - mse: 0.0456 - auc_16: 0.9817\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1597 - accuracy: 0.9410 - mae: 0.0915 - mse: 0.0454 - auc_16: 0.9820\n",
      "94\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1649 - accuracy: 0.9387 - mae: 0.0941 - mse: 0.0470 - auc_16: 0.9807\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1613 - accuracy: 0.9407 - mae: 0.0923 - mse: 0.0455 - auc_16: 0.9814\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1560 - accuracy: 0.9431 - mae: 0.0893 - mse: 0.0441 - auc_16: 0.9824\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1562 - accuracy: 0.9417 - mae: 0.0892 - mse: 0.0445 - auc_16: 0.9825\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1517 - accuracy: 0.9443 - mae: 0.0866 - mse: 0.0430 - auc_16: 0.9833\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1627 - accuracy: 0.9403 - mae: 0.0933 - mse: 0.0460 - auc_16: 0.9814\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1588 - accuracy: 0.9420 - mae: 0.0904 - mse: 0.0451 - auc_16: 0.9821\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1540 - accuracy: 0.9438 - mae: 0.0875 - mse: 0.0435 - auc_16: 0.9830\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1524 - accuracy: 0.9449 - mae: 0.0867 - mse: 0.0427 - auc_16: 0.9833\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1519 - accuracy: 0.9444 - mae: 0.0866 - mse: 0.0429 - auc_16: 0.9834\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "93\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1514 - accuracy: 0.9453 - mae: 0.0858 - mse: 0.0426 - auc_16: 0.9836\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1467 - accuracy: 0.9463 - mae: 0.0833 - mse: 0.0414 - auc_16: 0.9845\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1510 - accuracy: 0.9454 - mae: 0.0854 - mse: 0.0425 - auc_16: 0.9835\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1505 - accuracy: 0.9454 - mae: 0.0860 - mse: 0.0424 - auc_16: 0.9838\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1524 - accuracy: 0.9450 - mae: 0.0860 - mse: 0.0428 - auc_16: 0.9833\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1511 - accuracy: 0.9446 - mae: 0.0859 - mse: 0.0427 - auc_16: 0.9836\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1500 - accuracy: 0.9465 - mae: 0.0851 - mse: 0.0419 - auc_16: 0.9838\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1486 - accuracy: 0.9468 - mae: 0.0842 - mse: 0.0416 - auc_16: 0.9840\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 9s 10ms/step - loss: 0.1482 - accuracy: 0.9467 - mae: 0.0840 - mse: 0.0416 - auc_16: 0.9842\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 9s 11ms/step - loss: 0.1495 - accuracy: 0.9472 - mae: 0.0844 - mse: 0.0417 - auc_16: 0.9836\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "8\n",
      "244/244 [==============================] - 0s 2ms/step - loss: 0.3242 - accuracy: 0.9188 - mae: 0.0905 - mse: 0.0644 - auc_16: 0.8575\n",
      "train positive label: 5278 - train negative label: 64769\n",
      "up and down sampling => train positive label: 47502 - train negative label: 64769\n",
      "Test positive label: 575 - Test negative label: 7208\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.5834 - accuracy: 0.6940 - mae: 0.3935 - mse: 0.1972 - auc_17: 0.7586\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.4847 - accuracy: 0.7772 - mae: 0.3159 - mse: 0.1564 - auc_17: 0.8479\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.4452 - accuracy: 0.8001 - mae: 0.2880 - mse: 0.1421 - auc_17: 0.8735\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.4172 - accuracy: 0.8155 - mae: 0.2674 - mse: 0.1321 - auc_17: 0.8898\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.3926 - accuracy: 0.8270 - mae: 0.2509 - mse: 0.1237 - auc_17: 0.9029\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.3693 - accuracy: 0.8373 - mae: 0.2353 - mse: 0.1162 - auc_17: 0.9136\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.3530 - accuracy: 0.8460 - mae: 0.2235 - mse: 0.1104 - auc_17: 0.9211\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.3390 - accuracy: 0.8523 - mae: 0.2137 - mse: 0.1058 - auc_17: 0.9267\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.3265 - accuracy: 0.8606 - mae: 0.2051 - mse: 0.1013 - auc_17: 0.9318\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.3135 - accuracy: 0.8665 - mae: 0.1962 - mse: 0.0972 - auc_17: 0.9366\n",
      "86\n",
      "85\n",
      "84\n",
      "83\n",
      "82\n",
      "81\n",
      "80\n",
      "77\n",
      "69\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 12s 14ms/step - loss: 0.3053 - accuracy: 0.8710 - mae: 0.1906 - mse: 0.0943 - auc_17: 0.9397\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2976 - accuracy: 0.8737 - mae: 0.1848 - mse: 0.0917 - auc_17: 0.9429\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 12s 14ms/step - loss: 0.2908 - accuracy: 0.8784 - mae: 0.1801 - mse: 0.0891 - auc_17: 0.9448\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2814 - accuracy: 0.8833 - mae: 0.1735 - mse: 0.0859 - auc_17: 0.9482\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2756 - accuracy: 0.8866 - mae: 0.1694 - mse: 0.0840 - auc_17: 0.9502\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 12s 14ms/step - loss: 0.2704 - accuracy: 0.8879 - mae: 0.1659 - mse: 0.0823 - auc_17: 0.9520\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 12s 14ms/step - loss: 0.2664 - accuracy: 0.8901 - mae: 0.1634 - mse: 0.0808 - auc_17: 0.9532\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2559 - accuracy: 0.8949 - mae: 0.1560 - mse: 0.0776 - auc_17: 0.9566\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 12s 14ms/step - loss: 0.2530 - accuracy: 0.8967 - mae: 0.1543 - mse: 0.0764 - auc_17: 0.9574\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2480 - accuracy: 0.8992 - mae: 0.1508 - mse: 0.0744 - auc_17: 0.9594\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "87\n",
      "87\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2512 - accuracy: 0.8985 - mae: 0.1523 - mse: 0.0755 - auc_17: 0.9582\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2453 - accuracy: 0.9007 - mae: 0.1485 - mse: 0.0739 - auc_17: 0.9603\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2409 - accuracy: 0.9028 - mae: 0.1457 - mse: 0.0724 - auc_17: 0.9614\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2400 - accuracy: 0.9041 - mae: 0.1440 - mse: 0.0714 - auc_17: 0.9620\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2330 - accuracy: 0.9068 - mae: 0.1400 - mse: 0.0696 - auc_17: 0.9637\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2298 - accuracy: 0.9085 - mae: 0.1375 - mse: 0.0684 - auc_17: 0.9646\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2314 - accuracy: 0.9069 - mae: 0.1393 - mse: 0.0691 - auc_17: 0.9640\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2262 - accuracy: 0.9104 - mae: 0.1353 - mse: 0.0673 - auc_17: 0.9660\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2243 - accuracy: 0.9105 - mae: 0.1340 - mse: 0.0667 - auc_17: 0.9663\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2245 - accuracy: 0.9106 - mae: 0.1341 - mse: 0.0669 - auc_17: 0.9665\n",
      "91\n",
      "91\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "89\n",
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2180 - accuracy: 0.9128 - mae: 0.1300 - mse: 0.0648 - auc_17: 0.9679\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2129 - accuracy: 0.9155 - mae: 0.1269 - mse: 0.0630 - auc_17: 0.9693\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2167 - accuracy: 0.9128 - mae: 0.1295 - mse: 0.0644 - auc_17: 0.9683\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2128 - accuracy: 0.9156 - mae: 0.1264 - mse: 0.0628 - auc_17: 0.9695\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2134 - accuracy: 0.9161 - mae: 0.1269 - mse: 0.0630 - auc_17: 0.9694\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2036 - accuracy: 0.9210 - mae: 0.1204 - mse: 0.0598 - auc_17: 0.9718\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2028 - accuracy: 0.9203 - mae: 0.1205 - mse: 0.0597 - auc_17: 0.9720\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1977 - accuracy: 0.9231 - mae: 0.1166 - mse: 0.0580 - auc_17: 0.9736\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2018 - accuracy: 0.9219 - mae: 0.1189 - mse: 0.0589 - auc_17: 0.9726\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.2034 - accuracy: 0.9213 - mae: 0.1198 - mse: 0.0595 - auc_17: 0.9723\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1965 - accuracy: 0.9241 - mae: 0.1156 - mse: 0.0575 - auc_17: 0.9739\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1924 - accuracy: 0.9257 - mae: 0.1130 - mse: 0.0562 - auc_17: 0.9749\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1927 - accuracy: 0.9247 - mae: 0.1134 - mse: 0.0565 - auc_17: 0.9747\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1920 - accuracy: 0.9260 - mae: 0.1124 - mse: 0.0560 - auc_17: 0.9749\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1895 - accuracy: 0.9280 - mae: 0.1100 - mse: 0.0548 - auc_17: 0.9754\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1895 - accuracy: 0.9278 - mae: 0.1101 - mse: 0.0550 - auc_17: 0.9753\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1937 - accuracy: 0.9261 - mae: 0.1131 - mse: 0.0563 - auc_17: 0.9744\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1787 - accuracy: 0.9318 - mae: 0.1042 - mse: 0.0517 - auc_17: 0.9779\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1787 - accuracy: 0.9321 - mae: 0.1039 - mse: 0.0516 - auc_17: 0.9778\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1753 - accuracy: 0.9325 - mae: 0.1025 - mse: 0.0509 - auc_17: 0.9787\n",
      "93\n",
      "93\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1737 - accuracy: 0.9342 - mae: 0.1013 - mse: 0.0501 - auc_17: 0.9792\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1779 - accuracy: 0.9329 - mae: 0.1035 - mse: 0.0513 - auc_17: 0.9780\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1783 - accuracy: 0.9326 - mae: 0.1031 - mse: 0.0513 - auc_17: 0.9782\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1747 - accuracy: 0.9341 - mae: 0.1017 - mse: 0.0503 - auc_17: 0.9792\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1680 - accuracy: 0.9376 - mae: 0.0968 - mse: 0.0481 - auc_17: 0.9800\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1766 - accuracy: 0.9331 - mae: 0.1024 - mse: 0.0509 - auc_17: 0.9787\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1734 - accuracy: 0.9344 - mae: 0.1002 - mse: 0.0497 - auc_17: 0.9794\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1710 - accuracy: 0.9354 - mae: 0.0991 - mse: 0.0492 - auc_17: 0.9796\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1734 - accuracy: 0.9344 - mae: 0.1005 - mse: 0.0497 - auc_17: 0.9791\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 12s 13ms/step - loss: 0.1738 - accuracy: 0.9347 - mae: 0.1003 - mse: 0.0498 - auc_17: 0.9792\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "6\n",
      "244/244 [==============================] - 0s 2ms/step - loss: 0.2558 - accuracy: 0.9056 - mae: 0.1198 - mse: 0.0712 - auc_17: 0.8792\n",
      "train positive label: 5292 - train negative label: 64755\n",
      "up and down sampling => train positive label: 47628 - train negative label: 64755\n",
      "Test positive label: 561 - Test negative label: 7222\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 11s 12ms/step - loss: 0.5759 - accuracy: 0.7074 - mae: 0.3876 - mse: 0.1937 - auc_18: 0.7681\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 11s 12ms/step - loss: 0.4789 - accuracy: 0.7829 - mae: 0.3120 - mse: 0.1542 - auc_18: 0.8527\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 11s 12ms/step - loss: 0.4418 - accuracy: 0.8023 - mae: 0.2847 - mse: 0.1406 - auc_18: 0.8763\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.4128 - accuracy: 0.8157 - mae: 0.2640 - mse: 0.1307 - auc_18: 0.8925\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.3902 - accuracy: 0.8267 - mae: 0.2491 - mse: 0.1234 - auc_18: 0.9040\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.3700 - accuracy: 0.8367 - mae: 0.2354 - mse: 0.1164 - auc_18: 0.9137\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.3549 - accuracy: 0.8445 - mae: 0.2253 - mse: 0.1114 - auc_18: 0.9202\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.3390 - accuracy: 0.8516 - mae: 0.2142 - mse: 0.1061 - auc_18: 0.9269\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.3247 - accuracy: 0.8585 - mae: 0.2044 - mse: 0.1011 - auc_18: 0.9326\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.3139 - accuracy: 0.8662 - mae: 0.1966 - mse: 0.0971 - auc_18: 0.9371\n",
      "85\n",
      "85\n",
      "84\n",
      "83\n",
      "82\n",
      "81\n",
      "80\n",
      "78\n",
      "70\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.3056 - accuracy: 0.8693 - mae: 0.1912 - mse: 0.0947 - auc_18: 0.9396\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2986 - accuracy: 0.8735 - mae: 0.1861 - mse: 0.0920 - auc_18: 0.9425\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2875 - accuracy: 0.8796 - mae: 0.1783 - mse: 0.0882 - auc_18: 0.9460\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2802 - accuracy: 0.8830 - mae: 0.1734 - mse: 0.0857 - auc_18: 0.9486\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2717 - accuracy: 0.8881 - mae: 0.1677 - mse: 0.0829 - auc_18: 0.9512\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2671 - accuracy: 0.8899 - mae: 0.1645 - mse: 0.0812 - auc_18: 0.9531\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2572 - accuracy: 0.8950 - mae: 0.1575 - mse: 0.0779 - auc_18: 0.9563\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2500 - accuracy: 0.8986 - mae: 0.1527 - mse: 0.0756 - auc_18: 0.9583\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2472 - accuracy: 0.8994 - mae: 0.1506 - mse: 0.0746 - auc_18: 0.9590\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2428 - accuracy: 0.9021 - mae: 0.1473 - mse: 0.0730 - auc_18: 0.9602\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "87\n",
      "86\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2420 - accuracy: 0.9026 - mae: 0.1468 - mse: 0.0728 - auc_18: 0.9605\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2330 - accuracy: 0.9070 - mae: 0.1399 - mse: 0.0694 - auc_18: 0.9637\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2270 - accuracy: 0.9105 - mae: 0.1362 - mse: 0.0674 - auc_18: 0.9651\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2258 - accuracy: 0.9102 - mae: 0.1356 - mse: 0.0673 - auc_18: 0.9654\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2185 - accuracy: 0.9134 - mae: 0.1312 - mse: 0.0650 - auc_18: 0.9675\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2169 - accuracy: 0.9146 - mae: 0.1302 - mse: 0.0644 - auc_18: 0.9679\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2104 - accuracy: 0.9185 - mae: 0.1248 - mse: 0.0618 - auc_18: 0.9697\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2078 - accuracy: 0.9199 - mae: 0.1233 - mse: 0.0609 - auc_18: 0.9702\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2070 - accuracy: 0.9198 - mae: 0.1225 - mse: 0.0610 - auc_18: 0.9705\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2052 - accuracy: 0.9214 - mae: 0.1212 - mse: 0.0604 - auc_18: 0.9706\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "90\n",
      "90\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.2005 - accuracy: 0.9226 - mae: 0.1190 - mse: 0.0589 - auc_18: 0.9722\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1970 - accuracy: 0.9239 - mae: 0.1163 - mse: 0.0576 - auc_18: 0.9732\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1930 - accuracy: 0.9257 - mae: 0.1137 - mse: 0.0563 - auc_18: 0.9740\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1881 - accuracy: 0.9283 - mae: 0.1103 - mse: 0.0547 - auc_18: 0.9753\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1871 - accuracy: 0.9283 - mae: 0.1095 - mse: 0.0544 - auc_18: 0.9754\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1847 - accuracy: 0.9300 - mae: 0.1081 - mse: 0.0536 - auc_18: 0.9759\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1820 - accuracy: 0.9308 - mae: 0.1065 - mse: 0.0527 - auc_18: 0.9768\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1794 - accuracy: 0.9326 - mae: 0.1041 - mse: 0.0515 - auc_18: 0.9775\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1741 - accuracy: 0.9350 - mae: 0.1013 - mse: 0.0501 - auc_18: 0.9786\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1783 - accuracy: 0.9324 - mae: 0.1036 - mse: 0.0517 - auc_18: 0.9775\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1736 - accuracy: 0.9353 - mae: 0.1008 - mse: 0.0497 - auc_18: 0.9787\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1708 - accuracy: 0.9367 - mae: 0.0986 - mse: 0.0488 - auc_18: 0.9791\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1744 - accuracy: 0.9350 - mae: 0.1008 - mse: 0.0499 - auc_18: 0.9785\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1707 - accuracy: 0.9368 - mae: 0.0983 - mse: 0.0489 - auc_18: 0.9793\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1645 - accuracy: 0.9386 - mae: 0.0952 - mse: 0.0471 - auc_18: 0.9805\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1657 - accuracy: 0.9385 - mae: 0.0952 - mse: 0.0473 - auc_18: 0.9802\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1646 - accuracy: 0.9394 - mae: 0.0946 - mse: 0.0468 - auc_18: 0.9805\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1620 - accuracy: 0.9398 - mae: 0.0933 - mse: 0.0464 - auc_18: 0.9811\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1597 - accuracy: 0.9413 - mae: 0.0915 - mse: 0.0455 - auc_18: 0.9819\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1595 - accuracy: 0.9407 - mae: 0.0918 - mse: 0.0456 - auc_18: 0.9817\n",
      "94\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "Epoch 1/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1584 - accuracy: 0.9418 - mae: 0.0905 - mse: 0.0450 - auc_18: 0.9820\n",
      "Epoch 2/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1612 - accuracy: 0.9410 - mae: 0.0923 - mse: 0.0458 - auc_18: 0.9812\n",
      "Epoch 3/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1560 - accuracy: 0.9426 - mae: 0.0890 - mse: 0.0443 - auc_18: 0.9824\n",
      "Epoch 4/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1573 - accuracy: 0.9418 - mae: 0.0895 - mse: 0.0446 - auc_18: 0.9822\n",
      "Epoch 5/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1575 - accuracy: 0.9419 - mae: 0.0901 - mse: 0.0447 - auc_18: 0.9822\n",
      "Epoch 6/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1565 - accuracy: 0.9433 - mae: 0.0894 - mse: 0.0442 - auc_18: 0.9823\n",
      "Epoch 7/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1528 - accuracy: 0.9439 - mae: 0.0871 - mse: 0.0434 - auc_18: 0.9829\n",
      "Epoch 8/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1541 - accuracy: 0.9437 - mae: 0.0876 - mse: 0.0434 - auc_18: 0.9829\n",
      "Epoch 9/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1526 - accuracy: 0.9445 - mae: 0.0868 - mse: 0.0432 - auc_18: 0.9830\n",
      "Epoch 10/10\n",
      "878/878 [==============================] - 10s 12ms/step - loss: 0.1502 - accuracy: 0.9453 - mae: 0.0854 - mse: 0.0424 - auc_18: 0.9834\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "6\n",
      "244/244 [==============================] - 0s 2ms/step - loss: 0.3196 - accuracy: 0.9098 - mae: 0.1027 - mse: 0.0719 - auc_18: 0.8532\n",
      "train positive label: 5224 - train negative label: 64823\n",
      "up and down sampling => train positive label: 47016 - train negative label: 64823\n",
      "Test positive label: 629 - Test negative label: 7154\n",
      "Epoch 1/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.5690 - accuracy: 0.7133 - mae: 0.3815 - mse: 0.1909 - auc_19: 0.7738\n",
      "Epoch 2/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.4778 - accuracy: 0.7822 - mae: 0.3109 - mse: 0.1536 - auc_19: 0.8532\n",
      "Epoch 3/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.4395 - accuracy: 0.8027 - mae: 0.2829 - mse: 0.1397 - auc_19: 0.8775\n",
      "Epoch 4/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.4125 - accuracy: 0.8194 - mae: 0.2632 - mse: 0.1299 - auc_19: 0.8927\n",
      "Epoch 5/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.3871 - accuracy: 0.8300 - mae: 0.2465 - mse: 0.1219 - auc_19: 0.9054\n",
      "Epoch 6/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.3643 - accuracy: 0.8400 - mae: 0.2311 - mse: 0.1142 - auc_19: 0.9163\n",
      "Epoch 7/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.3468 - accuracy: 0.8498 - mae: 0.2190 - mse: 0.1082 - auc_19: 0.9239\n",
      "Epoch 8/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.3282 - accuracy: 0.8588 - mae: 0.2067 - mse: 0.1021 - auc_19: 0.9314\n",
      "Epoch 9/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.3175 - accuracy: 0.8638 - mae: 0.1992 - mse: 0.0983 - auc_19: 0.9357\n",
      "Epoch 10/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.3063 - accuracy: 0.8700 - mae: 0.1910 - mse: 0.0944 - auc_19: 0.9400\n",
      "86\n",
      "85\n",
      "84\n",
      "84\n",
      "82\n",
      "81\n",
      "80\n",
      "78\n",
      "71\n",
      "Epoch 1/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2940 - accuracy: 0.8752 - mae: 0.1830 - mse: 0.0905 - auc_19: 0.9446\n",
      "Epoch 2/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2844 - accuracy: 0.8808 - mae: 0.1759 - mse: 0.0869 - auc_19: 0.9482\n",
      "Epoch 3/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2787 - accuracy: 0.8842 - mae: 0.1723 - mse: 0.0848 - auc_19: 0.9501\n",
      "Epoch 4/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2655 - accuracy: 0.8895 - mae: 0.1637 - mse: 0.0807 - auc_19: 0.9545\n",
      "Epoch 5/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2596 - accuracy: 0.8940 - mae: 0.1584 - mse: 0.0784 - auc_19: 0.9561\n",
      "Epoch 6/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2541 - accuracy: 0.8966 - mae: 0.1551 - mse: 0.0768 - auc_19: 0.9580\n",
      "Epoch 7/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2441 - accuracy: 0.9013 - mae: 0.1481 - mse: 0.0731 - auc_19: 0.9610\n",
      "Epoch 8/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2398 - accuracy: 0.9025 - mae: 0.1457 - mse: 0.0721 - auc_19: 0.9620\n",
      "Epoch 9/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2305 - accuracy: 0.9076 - mae: 0.1395 - mse: 0.0689 - auc_19: 0.9647\n",
      "Epoch 10/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2310 - accuracy: 0.9075 - mae: 0.1391 - mse: 0.0687 - auc_19: 0.9650\n",
      "90\n",
      "90\n",
      "90\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "Epoch 1/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2252 - accuracy: 0.9102 - mae: 0.1355 - mse: 0.0671 - auc_19: 0.9663\n",
      "Epoch 2/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2186 - accuracy: 0.9128 - mae: 0.1308 - mse: 0.0648 - auc_19: 0.9680\n",
      "Epoch 3/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2147 - accuracy: 0.9145 - mae: 0.1288 - mse: 0.0637 - auc_19: 0.9690\n",
      "Epoch 4/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2119 - accuracy: 0.9166 - mae: 0.1265 - mse: 0.0625 - auc_19: 0.9699\n",
      "Epoch 5/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2091 - accuracy: 0.9190 - mae: 0.1241 - mse: 0.0614 - auc_19: 0.9705\n",
      "Epoch 6/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2111 - accuracy: 0.9163 - mae: 0.1252 - mse: 0.0625 - auc_19: 0.9702\n",
      "Epoch 7/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2039 - accuracy: 0.9204 - mae: 0.1209 - mse: 0.0599 - auc_19: 0.9718\n",
      "Epoch 8/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.2006 - accuracy: 0.9217 - mae: 0.1189 - mse: 0.0587 - auc_19: 0.9729\n",
      "Epoch 9/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1981 - accuracy: 0.9236 - mae: 0.1165 - mse: 0.0576 - auc_19: 0.9735\n",
      "Epoch 10/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1935 - accuracy: 0.9240 - mae: 0.1141 - mse: 0.0567 - auc_19: 0.9747\n",
      "92\n",
      "92\n",
      "92\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "Epoch 1/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1896 - accuracy: 0.9270 - mae: 0.1112 - mse: 0.0550 - auc_19: 0.9753\n",
      "Epoch 2/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1892 - accuracy: 0.9273 - mae: 0.1110 - mse: 0.0549 - auc_19: 0.9756\n",
      "Epoch 3/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1861 - accuracy: 0.9291 - mae: 0.1087 - mse: 0.0539 - auc_19: 0.9761\n",
      "Epoch 4/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1884 - accuracy: 0.9280 - mae: 0.1101 - mse: 0.0545 - auc_19: 0.9758\n",
      "Epoch 5/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1804 - accuracy: 0.9308 - mae: 0.1054 - mse: 0.0523 - auc_19: 0.9776\n",
      "Epoch 6/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1827 - accuracy: 0.9302 - mae: 0.1067 - mse: 0.0531 - auc_19: 0.9772\n",
      "Epoch 7/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1793 - accuracy: 0.9310 - mae: 0.1043 - mse: 0.0521 - auc_19: 0.9780\n",
      "Epoch 8/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1792 - accuracy: 0.9323 - mae: 0.1042 - mse: 0.0517 - auc_19: 0.9780\n",
      "Epoch 9/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1786 - accuracy: 0.9323 - mae: 0.1036 - mse: 0.0515 - auc_19: 0.9781\n",
      "Epoch 10/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1768 - accuracy: 0.9336 - mae: 0.1023 - mse: 0.0509 - auc_19: 0.9783\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "Epoch 1/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1719 - accuracy: 0.9345 - mae: 0.0996 - mse: 0.0494 - auc_19: 0.9798\n",
      "Epoch 2/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1683 - accuracy: 0.9369 - mae: 0.0972 - mse: 0.0480 - auc_19: 0.9804\n",
      "Epoch 3/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1726 - accuracy: 0.9349 - mae: 0.0995 - mse: 0.0496 - auc_19: 0.9794\n",
      "Epoch 4/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1711 - accuracy: 0.9347 - mae: 0.0989 - mse: 0.0491 - auc_19: 0.9799\n",
      "Epoch 5/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1646 - accuracy: 0.9385 - mae: 0.0946 - mse: 0.0468 - auc_19: 0.9810\n",
      "Epoch 6/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1574 - accuracy: 0.9407 - mae: 0.0904 - mse: 0.0450 - auc_19: 0.9825\n",
      "Epoch 7/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1639 - accuracy: 0.9389 - mae: 0.0941 - mse: 0.0468 - auc_19: 0.9810\n",
      "Epoch 8/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1639 - accuracy: 0.9392 - mae: 0.0940 - mse: 0.0466 - auc_19: 0.9813\n",
      "Epoch 9/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1608 - accuracy: 0.9396 - mae: 0.0926 - mse: 0.0459 - auc_19: 0.9820\n",
      "Epoch 10/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1615 - accuracy: 0.9399 - mae: 0.0925 - mse: 0.0457 - auc_19: 0.9818\n",
      "93\n",
      "93\n",
      "93\n",
      "94\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "Epoch 1/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1544 - accuracy: 0.9426 - mae: 0.0883 - mse: 0.0439 - auc_19: 0.9832\n",
      "Epoch 2/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1557 - accuracy: 0.9417 - mae: 0.0892 - mse: 0.0441 - auc_19: 0.9830\n",
      "Epoch 3/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1546 - accuracy: 0.9423 - mae: 0.0882 - mse: 0.0438 - auc_19: 0.9832\n",
      "Epoch 4/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1532 - accuracy: 0.9433 - mae: 0.0873 - mse: 0.0434 - auc_19: 0.9834\n",
      "Epoch 5/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1511 - accuracy: 0.9439 - mae: 0.0859 - mse: 0.0426 - auc_19: 0.9840\n",
      "Epoch 6/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1490 - accuracy: 0.9446 - mae: 0.0848 - mse: 0.0421 - auc_19: 0.9843\n",
      "Epoch 7/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1507 - accuracy: 0.9439 - mae: 0.0860 - mse: 0.0426 - auc_19: 0.9839\n",
      "Epoch 8/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1472 - accuracy: 0.9447 - mae: 0.0832 - mse: 0.0416 - auc_19: 0.9847\n",
      "Epoch 9/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1462 - accuracy: 0.9455 - mae: 0.0830 - mse: 0.0414 - auc_19: 0.9845\n",
      "Epoch 10/10\n",
      "874/874 [==============================] - 9s 11ms/step - loss: 0.1458 - accuracy: 0.9452 - mae: 0.0828 - mse: 0.0413 - auc_19: 0.9847\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "6\n",
      "244/244 [==============================] - 0s 2ms/step - loss: 0.3353 - accuracy: 0.9153 - mae: 0.0958 - mse: 0.0670 - auc_19: 0.8701\n",
      "train positive label: 5229 - train negative label: 64818\n",
      "up and down sampling => train positive label: 47061 - train negative label: 64818\n",
      "Test positive label: 624 - Test negative label: 7159\n",
      "Epoch 1/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.5603 - accuracy: 0.7176 - mae: 0.3760 - mse: 0.1877 - auc_20: 0.7825\n",
      "Epoch 2/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.4755 - accuracy: 0.7857 - mae: 0.3102 - mse: 0.1532 - auc_20: 0.8539\n",
      "Epoch 3/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.4331 - accuracy: 0.8080 - mae: 0.2797 - mse: 0.1379 - auc_20: 0.8806\n",
      "Epoch 4/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.4003 - accuracy: 0.8233 - mae: 0.2562 - mse: 0.1264 - auc_20: 0.8987\n",
      "Epoch 5/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.3775 - accuracy: 0.8326 - mae: 0.2410 - mse: 0.1191 - auc_20: 0.9097\n",
      "Epoch 6/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.3543 - accuracy: 0.8468 - mae: 0.2246 - mse: 0.1109 - auc_20: 0.9200\n",
      "Epoch 7/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.3369 - accuracy: 0.8552 - mae: 0.2128 - mse: 0.1050 - auc_20: 0.9269\n",
      "Epoch 8/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.3234 - accuracy: 0.8605 - mae: 0.2038 - mse: 0.1007 - auc_20: 0.9328\n",
      "Epoch 9/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.3097 - accuracy: 0.8678 - mae: 0.1942 - mse: 0.0960 - auc_20: 0.9374\n",
      "Epoch 10/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.3009 - accuracy: 0.8733 - mae: 0.1880 - mse: 0.0931 - auc_20: 0.9406\n",
      "86\n",
      "86\n",
      "85\n",
      "84\n",
      "83\n",
      "82\n",
      "80\n",
      "78\n",
      "71\n",
      "Epoch 1/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2937 - accuracy: 0.8755 - mae: 0.1832 - mse: 0.0907 - auc_20: 0.9433\n",
      "Epoch 2/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2824 - accuracy: 0.8815 - mae: 0.1750 - mse: 0.0867 - auc_20: 0.9473\n",
      "Epoch 3/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2766 - accuracy: 0.8855 - mae: 0.1715 - mse: 0.0845 - auc_20: 0.9491\n",
      "Epoch 4/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2702 - accuracy: 0.8888 - mae: 0.1664 - mse: 0.0822 - auc_20: 0.9515\n",
      "Epoch 5/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2608 - accuracy: 0.8939 - mae: 0.1604 - mse: 0.0793 - auc_20: 0.9544\n",
      "Epoch 6/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2559 - accuracy: 0.8952 - mae: 0.1566 - mse: 0.0776 - auc_20: 0.9564\n",
      "Epoch 7/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2528 - accuracy: 0.8969 - mae: 0.1544 - mse: 0.0765 - auc_20: 0.9572\n",
      "Epoch 8/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2496 - accuracy: 0.8986 - mae: 0.1516 - mse: 0.0751 - auc_20: 0.9581\n",
      "Epoch 9/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2426 - accuracy: 0.9025 - mae: 0.1475 - mse: 0.0731 - auc_20: 0.9601\n",
      "Epoch 10/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2355 - accuracy: 0.9057 - mae: 0.1427 - mse: 0.0705 - auc_20: 0.9626\n",
      "90\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "Epoch 1/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2334 - accuracy: 0.9069 - mae: 0.1403 - mse: 0.0697 - auc_20: 0.9631\n",
      "Epoch 2/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2294 - accuracy: 0.9094 - mae: 0.1377 - mse: 0.0682 - auc_20: 0.9644\n",
      "Epoch 3/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2232 - accuracy: 0.9116 - mae: 0.1341 - mse: 0.0663 - auc_20: 0.9661\n",
      "Epoch 4/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2194 - accuracy: 0.9140 - mae: 0.1312 - mse: 0.0651 - auc_20: 0.9669\n",
      "Epoch 5/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2154 - accuracy: 0.9156 - mae: 0.1288 - mse: 0.0637 - auc_20: 0.9681\n",
      "Epoch 6/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2130 - accuracy: 0.9172 - mae: 0.1262 - mse: 0.0627 - auc_20: 0.9688\n",
      "Epoch 7/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2138 - accuracy: 0.9166 - mae: 0.1271 - mse: 0.0630 - auc_20: 0.9687\n",
      "Epoch 8/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2076 - accuracy: 0.9192 - mae: 0.1231 - mse: 0.0611 - auc_20: 0.9705\n",
      "Epoch 9/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2045 - accuracy: 0.9201 - mae: 0.1214 - mse: 0.0603 - auc_20: 0.9711\n",
      "Epoch 10/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2026 - accuracy: 0.9217 - mae: 0.1195 - mse: 0.0595 - auc_20: 0.9716\n",
      "92\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "91\n",
      "90\n",
      "90\n",
      "Epoch 1/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.2027 - accuracy: 0.9211 - mae: 0.1200 - mse: 0.0595 - auc_20: 0.9720\n",
      "Epoch 2/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1957 - accuracy: 0.9248 - mae: 0.1153 - mse: 0.0573 - auc_20: 0.9734\n",
      "Epoch 3/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1921 - accuracy: 0.9272 - mae: 0.1127 - mse: 0.0559 - auc_20: 0.9743\n",
      "Epoch 4/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1926 - accuracy: 0.9266 - mae: 0.1128 - mse: 0.0561 - auc_20: 0.9744\n",
      "Epoch 5/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1859 - accuracy: 0.9293 - mae: 0.1085 - mse: 0.0539 - auc_20: 0.9755\n",
      "Epoch 6/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1850 - accuracy: 0.9300 - mae: 0.1081 - mse: 0.0535 - auc_20: 0.9759\n",
      "Epoch 7/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1888 - accuracy: 0.9287 - mae: 0.1099 - mse: 0.0545 - auc_20: 0.9749\n",
      "Epoch 8/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1846 - accuracy: 0.9298 - mae: 0.1079 - mse: 0.0535 - auc_20: 0.9760\n",
      "Epoch 9/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1788 - accuracy: 0.9328 - mae: 0.1042 - mse: 0.0515 - auc_20: 0.9775\n",
      "Epoch 10/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1805 - accuracy: 0.9325 - mae: 0.1040 - mse: 0.0518 - auc_20: 0.9770\n",
      "93\n",
      "92\n",
      "92\n",
      "93\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "92\n",
      "Epoch 1/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1771 - accuracy: 0.9338 - mae: 0.1025 - mse: 0.0510 - auc_20: 0.9776\n",
      "Epoch 2/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1784 - accuracy: 0.9328 - mae: 0.1035 - mse: 0.0515 - auc_20: 0.9774\n",
      "Epoch 3/10\n",
      "875/875 [==============================] - 10s 11ms/step - loss: 0.1775 - accuracy: 0.9335 - mae: 0.1026 - mse: 0.0510 - auc_20: 0.9780\n",
      "Epoch 4/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1744 - accuracy: 0.9353 - mae: 0.1006 - mse: 0.0498 - auc_20: 0.9785\n",
      "Epoch 5/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1729 - accuracy: 0.9362 - mae: 0.0995 - mse: 0.0493 - auc_20: 0.9785\n",
      "Epoch 6/10\n",
      "875/875 [==============================] - 10s 11ms/step - loss: 0.1677 - accuracy: 0.9381 - mae: 0.0965 - mse: 0.0477 - auc_20: 0.9800\n",
      "Epoch 7/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1701 - accuracy: 0.9372 - mae: 0.0974 - mse: 0.0486 - auc_20: 0.9793\n",
      "Epoch 8/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1671 - accuracy: 0.9379 - mae: 0.0960 - mse: 0.0477 - auc_20: 0.9801\n",
      "Epoch 9/10\n",
      "875/875 [==============================] - 10s 11ms/step - loss: 0.1614 - accuracy: 0.9409 - mae: 0.0927 - mse: 0.0458 - auc_20: 0.9811\n",
      "Epoch 10/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1628 - accuracy: 0.9402 - mae: 0.0928 - mse: 0.0462 - auc_20: 0.9809\n",
      "94\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "93\n",
      "Epoch 1/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1645 - accuracy: 0.9393 - mae: 0.0940 - mse: 0.0468 - auc_20: 0.9807\n",
      "Epoch 2/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1640 - accuracy: 0.9392 - mae: 0.0939 - mse: 0.0466 - auc_20: 0.9810\n",
      "Epoch 3/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1584 - accuracy: 0.9416 - mae: 0.0909 - mse: 0.0451 - auc_20: 0.9817\n",
      "Epoch 4/10\n",
      "875/875 [==============================] - 10s 11ms/step - loss: 0.1593 - accuracy: 0.9420 - mae: 0.0908 - mse: 0.0452 - auc_20: 0.9815\n",
      "Epoch 5/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1549 - accuracy: 0.9438 - mae: 0.0888 - mse: 0.0439 - auc_20: 0.9825\n",
      "Epoch 6/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1546 - accuracy: 0.9432 - mae: 0.0883 - mse: 0.0438 - auc_20: 0.9826\n",
      "Epoch 7/10\n",
      "875/875 [==============================] - 10s 11ms/step - loss: 0.1548 - accuracy: 0.9431 - mae: 0.0885 - mse: 0.0438 - auc_20: 0.9827\n",
      "Epoch 8/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1547 - accuracy: 0.9431 - mae: 0.0883 - mse: 0.0439 - auc_20: 0.9827\n",
      "Epoch 9/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1522 - accuracy: 0.9447 - mae: 0.0864 - mse: 0.0430 - auc_20: 0.9828\n",
      "Epoch 10/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1550 - accuracy: 0.9441 - mae: 0.0881 - mse: 0.0436 - auc_20: 0.9825\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "93\n",
      "93\n",
      "Epoch 1/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1497 - accuracy: 0.9462 - mae: 0.0846 - mse: 0.0421 - auc_20: 0.9833\n",
      "Epoch 2/10\n",
      "875/875 [==============================] - 10s 11ms/step - loss: 0.1502 - accuracy: 0.9454 - mae: 0.0853 - mse: 0.0424 - auc_20: 0.9835\n",
      "Epoch 3/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1501 - accuracy: 0.9461 - mae: 0.0855 - mse: 0.0421 - auc_20: 0.9834\n",
      "Epoch 4/10\n",
      "875/875 [==============================] - 10s 11ms/step - loss: 0.1498 - accuracy: 0.9460 - mae: 0.0849 - mse: 0.0421 - auc_20: 0.9833\n",
      "Epoch 5/10\n",
      "875/875 [==============================] - 10s 11ms/step - loss: 0.1480 - accuracy: 0.9467 - mae: 0.0837 - mse: 0.0416 - auc_20: 0.9840\n",
      "Epoch 6/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1424 - accuracy: 0.9490 - mae: 0.0801 - mse: 0.0398 - auc_20: 0.9848\n",
      "Epoch 7/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1448 - accuracy: 0.9484 - mae: 0.0816 - mse: 0.0404 - auc_20: 0.9844\n",
      "Epoch 8/10\n",
      "875/875 [==============================] - 10s 11ms/step - loss: 0.1408 - accuracy: 0.9499 - mae: 0.0795 - mse: 0.0393 - auc_20: 0.9849\n",
      "Epoch 9/10\n",
      "875/875 [==============================] - 10s 12ms/step - loss: 0.1412 - accuracy: 0.9495 - mae: 0.0796 - mse: 0.0393 - auc_20: 0.9849\n",
      "Epoch 10/10\n",
      "875/875 [==============================] - 10s 11ms/step - loss: 0.1427 - accuracy: 0.9493 - mae: 0.0806 - mse: 0.0398 - auc_20: 0.9848\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "94\n",
      "7\n",
      "244/244 [==============================] - 0s 2ms/step - loss: 0.2998 - accuracy: 0.9143 - mae: 0.1013 - mse: 0.0688 - auc_20: 0.8708\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "Epoch_S = 10\n",
    "\n",
    "def evaluate_model(dataset, subscriber_dataset, k = 10 , shuffle = False):\n",
    "    result =[]\n",
    "\n",
    "    kf = KFold(n_splits=10, shuffle= shuffle, random_state=None)\n",
    "\n",
    "    for train_index, test_index in kf.split(dataset):\n",
    "\n",
    "        train_ds = [dataset[index] for index in train_index]\n",
    "\n",
    "        valid_ds = [dataset[index] for index in test_index]\n",
    "\n",
    "        label_pos , label_neg, _ , _ = count_lablel(train_ds)\n",
    "        print(f'train positive label: {label_pos} - train negative label: {label_neg}')\n",
    "\n",
    "        train_ds = up_and_down_Samplenig(train_ds, scale_downsampling = 0.5)\n",
    "\n",
    "        label_pos , label_neg , _ , _ = count_lablel(train_ds)\n",
    "        print(f'up and down sampling => train positive label: {label_pos} - train negative label: {label_neg}')\n",
    "\n",
    "        label_pos , label_neg, _ , _ = count_lablel(valid_ds)\n",
    "        print(f'Test positive label: {label_pos} - Test negative label: {label_neg}')\n",
    "\n",
    "        l_train = []\n",
    "        r_train = []\n",
    "        lbls_train = []\n",
    "        l_valid = []\n",
    "        r_valid = []\n",
    "        lbls_valid = []\n",
    "\n",
    "        for i , data in enumerate(train_ds):\n",
    "            smiles, embbed_drug, onehot_task, embbed_task, lbl, task_name = data\n",
    "            l_train.append(embbed_drug[0])\n",
    "            r_train.append(embbed_task)\n",
    "            lbls_train.append(lbl.tolist())\n",
    "\n",
    "        for i , data in enumerate(valid_ds):\n",
    "            smiles, embbed_drug, onehot_task, embbed_task, lbl, task_name = data\n",
    "            l_valid.append(embbed_drug[0])\n",
    "            r_valid.append(embbed_task)\n",
    "            lbls_valid.append(lbl.tolist())\n",
    "\n",
    "        l_train = np.array(l_train).reshape(-1,1024,1)\n",
    "        r_train = np.array(r_train).reshape(-1,512,1)\n",
    "        lbls_train = np.array(lbls_train)\n",
    "\n",
    "        l_valid = np.array(l_valid).reshape(-1,1024,1)\n",
    "        r_valid = np.array(r_valid).reshape(-1,512,1)\n",
    "        lbls_valid = np.array(lbls_valid)\n",
    "\n",
    "        # create neural network model\n",
    "        siamese_net = siamese_model_attentiveFp_sider()\n",
    "        \n",
    "        history = History()\n",
    "        P = siamese_net.fit([l_train, r_train], lbls_train, epochs = Epoch_S, batch_size = 128, callbacks=[history])\n",
    "\n",
    "        for j in range(100):\n",
    "            C=1\n",
    "            Before = int(P.history['accuracy'][-1]*100)\n",
    "            for i in range(2,Epoch_S+1):\n",
    "                if  int(P.history['accuracy'][-i]*100) == Before:\n",
    "                    C=C+1\n",
    "                else:\n",
    "                    C=1\n",
    "                Before=int(P.history['accuracy'][-i]*100)\n",
    "                print(Before)\n",
    "            if C==Epoch_S:\n",
    "                break\n",
    "            P = siamese_net.fit([l_train, r_train], lbls_train, epochs = Epoch_S, batch_size = 128, callbacks=[history])\n",
    "        print(j+1)\n",
    "\n",
    "        score  = siamese_net.evaluate([l_valid,r_valid], lbls_valid, verbose=1)\n",
    "        a = (score[1],score[4])\n",
    "        result.append(a)\n",
    "\n",
    "    return result\n",
    "\n",
    "scores = evaluate_model(data_ds, subscriber_data_ds, 10, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true,
    "id": "IeKCR3RC8Q8m"
   },
   "source": [
    "#### Dropout = 0.3 and downsampling = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "3eTF0M3C8Q8m",
    "outputId": "846445cd-a178-468b-cdc5-e8ba6115bc23"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.909417986869812, 0.8909998536109924),\n",
       " (0.9104458689689636, 0.8691753149032593),\n",
       " (0.913144052028656, 0.8680707812309265),\n",
       " (0.9166131019592285, 0.8781477212905884),\n",
       " (0.920724630355835, 0.861714243888855),\n",
       " (0.9145573973655701, 0.8755292892456055),\n",
       " (0.9218810200691223, 0.8655411601066589),\n",
       " (0.9247077107429504, 0.8602591156959534),\n",
       " (0.922137975692749, 0.8681714534759521),\n",
       " (0.9125016331672668, 0.873445987701416)]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "TUMjkCyW8Q8m",
    "outputId": "f55d09e1-9df0-437d-f886-576a1d295519"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy= 0.9166131377220154 AUC= 0.8711054921150208 STD_AUC= 0.008536810669444152\n"
     ]
    }
   ],
   "source": [
    "acc = []\n",
    "auc = []\n",
    "for i in scores:\n",
    "    acc.append(i[0])\n",
    "    auc.append(i[1])\n",
    "\n",
    "print(f'accuracy= {np.mean(acc)} AUC= {np.mean(auc)} STD_AUC= {np.std(auc)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "phkc0iNd80_j"
   },
   "source": [
    "# **Case study with BioAct-Het**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8D1Dubhg9NIy"
   },
   "outputs": [],
   "source": [
    "model_name = 'GCN_attentivefp_SIDER'\n",
    "gcn_model = get_sider_model(model_name)\n",
    "gcn_model.eval()\n",
    "gcn_model = gcn_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-kdF95-S8EV3"
   },
   "outputs": [],
   "source": [
    "sider_smiles = df.smiles.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nL_MmnnA8EV3"
   },
   "outputs": [],
   "source": [
    "dir_path = 'C:/Users/Ali/Desktop/thesis'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HkRxOJiL8EV3"
   },
   "outputs": [],
   "source": [
    "df_case_study = pd.read_csv(dir_path + '/(sider)case_study.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0mJ-Pbw48EV3",
    "outputId": "8c8a04f5-c148-4954-bc9e-db6d190d6f05",
    "run_control": {
     "marked": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>smiles</th>\n",
       "      <th>Hepatobiliary disorders</th>\n",
       "      <th>Metabolism and nutrition disorders</th>\n",
       "      <th>Product issues</th>\n",
       "      <th>Eye disorders</th>\n",
       "      <th>Investigations</th>\n",
       "      <th>Musculoskeletal and connective tissue disorders</th>\n",
       "      <th>Gastrointestinal disorders</th>\n",
       "      <th>Social circumstances</th>\n",
       "      <th>...</th>\n",
       "      <th>Infections and infestations</th>\n",
       "      <th>Respiratory, thoracic and mediastinal disorders</th>\n",
       "      <th>Psychiatric disorders</th>\n",
       "      <th>Renal and urinary disorders</th>\n",
       "      <th>Pregnancy, puerperium and perinatal conditions</th>\n",
       "      <th>Ear and labyrinth disorders</th>\n",
       "      <th>Cardiac disorders</th>\n",
       "      <th>Nervous system disorders</th>\n",
       "      <th>Injury, poisoning and procedural complications</th>\n",
       "      <th>Drug_Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Cl.CN[C@H](CC(C)C)C(=O)N[C@@H]1[C@H](O)C2=CC=C...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Vancomycin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>CC[C@@H]1NC(=O)[C@H]([C@H](O)[C@H](C)C\\C=C\\C)N...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cyclosporine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Cl.CCCCCCCCC1=CC=C(CCC(N)(CO)CO)C=C1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fingolimod</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>CC(C)CC(C(=NC(CCC(=O)O)C(=NC(CCCCN)C(=NC(CCC(=...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>interferon-beta 1a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>CCC(CC)COC(=O)[C@H](C)N[P@](=O)(OC[C@H]1O[C@](...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Remdesivir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>CCOC(=O)C1=C[C@@H](OC(CC)CC)[C@H](NC(C)=O)[C@@...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Oseltamivir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>CC(C)[C@H](NC(=O)N(C)CC1=CSC(=N1)C(C)C)C(=O)N[...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ritonavir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>CC(C)C(=O)OC[C@H]1O[C@H]([C@H](O)[C@@H]1O)N1C=...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Molnupiravir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>CC1(C2C1C(N(C2)C(=O)C(C(C)(C)C)NC(=O)C(F)(F)F)...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Paxlovid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>CCC(N[C@H]1C2=CN=CC(C3=CC=C(N(C(CC4)=O)C)C4=C3...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Baxdrostat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>NC(=N)NC(=O)CC1=C(Cl)C=CC=C1Cl</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Guanfacine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>CCCCCCCCCCCCCCCC(=O)NC(CCC(=O)NCCCCC(C(=O)NC(C...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Liraglutide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>CC#CCN1C(=NC2=C1C(=O)N(CC1=NC3=C(C=CC=C3)C(C)=...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Linagliptin</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0                                             smiles  \\\n",
       "0            0  Cl.CN[C@H](CC(C)C)C(=O)N[C@@H]1[C@H](O)C2=CC=C...   \n",
       "1            1  CC[C@@H]1NC(=O)[C@H]([C@H](O)[C@H](C)C\\C=C\\C)N...   \n",
       "2            2               Cl.CCCCCCCCC1=CC=C(CCC(N)(CO)CO)C=C1   \n",
       "3            3  CC(C)CC(C(=NC(CCC(=O)O)C(=NC(CCCCN)C(=NC(CCC(=...   \n",
       "4            4  CCC(CC)COC(=O)[C@H](C)N[P@](=O)(OC[C@H]1O[C@](...   \n",
       "5            5  CCOC(=O)C1=C[C@@H](OC(CC)CC)[C@H](NC(C)=O)[C@@...   \n",
       "6            6  CC(C)[C@H](NC(=O)N(C)CC1=CSC(=N1)C(C)C)C(=O)N[...   \n",
       "7            7  CC(C)C(=O)OC[C@H]1O[C@H]([C@H](O)[C@@H]1O)N1C=...   \n",
       "8            8  CC1(C2C1C(N(C2)C(=O)C(C(C)(C)C)NC(=O)C(F)(F)F)...   \n",
       "9            9  CCC(N[C@H]1C2=CN=CC(C3=CC=C(N(C(CC4)=O)C)C4=C3...   \n",
       "10          10                     NC(=N)NC(=O)CC1=C(Cl)C=CC=C1Cl   \n",
       "11          11  CCCCCCCCCCCCCCCC(=O)NC(CCC(=O)NCCCCC(C(=O)NC(C...   \n",
       "12          12  CC#CCN1C(=NC2=C1C(=O)N(CC1=NC3=C(C=CC=C3)C(C)=...   \n",
       "\n",
       "    Hepatobiliary disorders  Metabolism and nutrition disorders  \\\n",
       "0                       NaN                                 NaN   \n",
       "1                       NaN                                 NaN   \n",
       "2                       NaN                                 NaN   \n",
       "3                       NaN                                 NaN   \n",
       "4                       NaN                                 NaN   \n",
       "5                       NaN                                 NaN   \n",
       "6                       NaN                                 NaN   \n",
       "7                       NaN                                 NaN   \n",
       "8                       NaN                                 NaN   \n",
       "9                       NaN                                 NaN   \n",
       "10                      NaN                                 NaN   \n",
       "11                      NaN                                 NaN   \n",
       "12                      NaN                                 NaN   \n",
       "\n",
       "    Product issues  Eye disorders  Investigations  \\\n",
       "0              NaN            NaN             NaN   \n",
       "1              NaN            NaN             NaN   \n",
       "2              NaN            NaN             NaN   \n",
       "3              NaN            NaN             NaN   \n",
       "4              NaN            NaN             NaN   \n",
       "5              NaN            NaN             NaN   \n",
       "6              NaN            NaN             NaN   \n",
       "7              NaN            NaN             NaN   \n",
       "8              NaN            NaN             NaN   \n",
       "9              NaN            NaN             NaN   \n",
       "10             NaN            NaN             NaN   \n",
       "11             NaN            NaN             NaN   \n",
       "12             NaN            NaN             NaN   \n",
       "\n",
       "    Musculoskeletal and connective tissue disorders  \\\n",
       "0                                               NaN   \n",
       "1                                               NaN   \n",
       "2                                               NaN   \n",
       "3                                               NaN   \n",
       "4                                               NaN   \n",
       "5                                               NaN   \n",
       "6                                               NaN   \n",
       "7                                               NaN   \n",
       "8                                               NaN   \n",
       "9                                               NaN   \n",
       "10                                              NaN   \n",
       "11                                              NaN   \n",
       "12                                              NaN   \n",
       "\n",
       "    Gastrointestinal disorders  Social circumstances  ...  \\\n",
       "0                          NaN                   NaN  ...   \n",
       "1                          NaN                   NaN  ...   \n",
       "2                          NaN                   NaN  ...   \n",
       "3                          NaN                   NaN  ...   \n",
       "4                          NaN                   NaN  ...   \n",
       "5                          NaN                   NaN  ...   \n",
       "6                          NaN                   NaN  ...   \n",
       "7                          NaN                   NaN  ...   \n",
       "8                          NaN                   NaN  ...   \n",
       "9                          NaN                   NaN  ...   \n",
       "10                         NaN                   NaN  ...   \n",
       "11                         NaN                   NaN  ...   \n",
       "12                         NaN                   NaN  ...   \n",
       "\n",
       "    Infections and infestations  \\\n",
       "0                           NaN   \n",
       "1                           NaN   \n",
       "2                           NaN   \n",
       "3                           NaN   \n",
       "4                           NaN   \n",
       "5                           NaN   \n",
       "6                           NaN   \n",
       "7                           NaN   \n",
       "8                           NaN   \n",
       "9                           NaN   \n",
       "10                          NaN   \n",
       "11                          NaN   \n",
       "12                          NaN   \n",
       "\n",
       "    Respiratory, thoracic and mediastinal disorders  Psychiatric disorders  \\\n",
       "0                                               NaN                    NaN   \n",
       "1                                               NaN                    NaN   \n",
       "2                                               NaN                    NaN   \n",
       "3                                               NaN                    NaN   \n",
       "4                                               NaN                    NaN   \n",
       "5                                               NaN                    NaN   \n",
       "6                                               NaN                    NaN   \n",
       "7                                               NaN                    NaN   \n",
       "8                                               NaN                    NaN   \n",
       "9                                               NaN                    NaN   \n",
       "10                                              NaN                    NaN   \n",
       "11                                              NaN                    NaN   \n",
       "12                                              NaN                    NaN   \n",
       "\n",
       "    Renal and urinary disorders  \\\n",
       "0                           NaN   \n",
       "1                           NaN   \n",
       "2                           NaN   \n",
       "3                           NaN   \n",
       "4                           NaN   \n",
       "5                           NaN   \n",
       "6                           NaN   \n",
       "7                           NaN   \n",
       "8                           NaN   \n",
       "9                           NaN   \n",
       "10                          NaN   \n",
       "11                          NaN   \n",
       "12                          NaN   \n",
       "\n",
       "    Pregnancy, puerperium and perinatal conditions  \\\n",
       "0                                              NaN   \n",
       "1                                              NaN   \n",
       "2                                              NaN   \n",
       "3                                              NaN   \n",
       "4                                              NaN   \n",
       "5                                              NaN   \n",
       "6                                              NaN   \n",
       "7                                              NaN   \n",
       "8                                              NaN   \n",
       "9                                              NaN   \n",
       "10                                             NaN   \n",
       "11                                             NaN   \n",
       "12                                             NaN   \n",
       "\n",
       "    Ear and labyrinth disorders  Cardiac disorders  Nervous system disorders  \\\n",
       "0                           NaN                NaN                       NaN   \n",
       "1                           NaN                NaN                       NaN   \n",
       "2                           NaN                NaN                       NaN   \n",
       "3                           NaN                NaN                       NaN   \n",
       "4                           NaN                NaN                       NaN   \n",
       "5                           NaN                NaN                       NaN   \n",
       "6                           NaN                NaN                       NaN   \n",
       "7                           NaN                NaN                       NaN   \n",
       "8                           NaN                NaN                       NaN   \n",
       "9                           NaN                NaN                       NaN   \n",
       "10                          NaN                NaN                       NaN   \n",
       "11                          NaN                NaN                       NaN   \n",
       "12                          NaN                NaN                       NaN   \n",
       "\n",
       "    Injury, poisoning and procedural complications           Drug_Name  \n",
       "0                                              NaN          Vancomycin  \n",
       "1                                              NaN        cyclosporine  \n",
       "2                                              NaN          fingolimod  \n",
       "3                                              NaN  interferon-beta 1a  \n",
       "4                                              NaN          Remdesivir  \n",
       "5                                              NaN         Oseltamivir  \n",
       "6                                              NaN          Ritonavir   \n",
       "7                                              NaN       Molnupiravir   \n",
       "8                                              NaN           Paxlovid   \n",
       "9                                              NaN          Baxdrostat  \n",
       "10                                             NaN         Guanfacine   \n",
       "11                                             NaN         Liraglutide  \n",
       "12                                             NaN         Linagliptin  \n",
       "\n",
       "[13 rows x 30 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_case_study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ih-Rji9o8EV3",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drug_name = df_case_study.Drug_Name.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "60K2ayNI8EV3"
   },
   "outputs": [],
   "source": [
    "candidate_smiles = df_case_study.smiles.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RSx0HEA08EV3",
    "outputId": "f8ec4cb4-4088-47e4-e2dc-62148dc15655"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "is_Membership(sider_smiles, candidate_smiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GNg7atocLsEl",
    "outputId": "221ddc8d-b5a3-4ebd-a5aa-dd98754c0f5a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/1427\n",
      "Data created!!\n"
     ]
    }
   ],
   "source": [
    "dataset = DATASET(df,smiles_to_bigraph, AttentiveFPAtomFeaturizer(), cache_file_path = cache_path_sider) \n",
    "ds_train = create_dataset_with_gcn_case_study(dataset, embed_class_sider, gcn_model, sider_tasks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vgkaBHkM8EV4",
    "outputId": "c8dfc821-96d0-4ae1-e826-daee89ee5bf1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgl graphs from scratch...\n",
      "Data created!!\n"
     ]
    }
   ],
   "source": [
    "dataset_study = DATASET(df_case_study[df_case_study.columns[1:29]],smiles_to_bigraph, \n",
    "                        AttentiveFPAtomFeaturizer(), cache_file_path = cache_path_sider)\n",
    "\n",
    "ds_study = create_dataset_with_gcn(dataset_study, embed_class_sider, gcn_model, sider_tasks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8Y_XgSN38EV4",
    "outputId": "83063539-b2e0-4085-d3f5-ef53d3c8ccfe"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "351"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ds_study)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NnpEfO3Y8EV4"
   },
   "source": [
    "### Training algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cVwfHwPH8EV4",
    "outputId": "9a54c6ef-4207-4b96-eb54-e4d69580f1b8",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.5382 - accuracy: 0.7347 - mae: 0.3561 - mse: 0.1788 - auc: 0.7999\n",
      "Epoch 2/15\n",
      "302/302 [==============================] - 3s 10ms/step - loss: 0.4719 - accuracy: 0.7867 - mae: 0.3057 - mse: 0.1523 - auc: 0.8556\n",
      "Epoch 3/15\n",
      "302/302 [==============================] - 3s 10ms/step - loss: 0.4464 - accuracy: 0.7958 - mae: 0.2880 - mse: 0.1433 - auc: 0.8718\n",
      "Epoch 4/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.4267 - accuracy: 0.8058 - mae: 0.2746 - mse: 0.1364 - auc: 0.8838\n",
      "Epoch 5/15\n",
      "302/302 [==============================] - 3s 10ms/step - loss: 0.4146 - accuracy: 0.8104 - mae: 0.2670 - mse: 0.1326 - auc: 0.8905\n",
      "Epoch 6/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.4113 - accuracy: 0.8144 - mae: 0.2644 - mse: 0.1310 - auc: 0.8928\n",
      "Epoch 7/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.4073 - accuracy: 0.8129 - mae: 0.2619 - mse: 0.1300 - auc: 0.8947\n",
      "Epoch 8/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.4003 - accuracy: 0.8152 - mae: 0.2572 - mse: 0.1280 - auc: 0.8980\n",
      "Epoch 9/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3970 - accuracy: 0.8193 - mae: 0.2550 - mse: 0.1264 - auc: 0.9003\n",
      "Epoch 10/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3964 - accuracy: 0.8193 - mae: 0.2546 - mse: 0.1263 - auc: 0.9007\n",
      "Epoch 11/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3911 - accuracy: 0.8215 - mae: 0.2508 - mse: 0.1246 - auc: 0.9034\n",
      "Epoch 12/15\n",
      "302/302 [==============================] - 3s 10ms/step - loss: 0.3864 - accuracy: 0.8246 - mae: 0.2477 - mse: 0.1229 - auc: 0.9057\n",
      "Epoch 13/15\n",
      "302/302 [==============================] - 3s 10ms/step - loss: 0.3843 - accuracy: 0.8250 - mae: 0.2469 - mse: 0.1224 - auc: 0.9066\n",
      "Epoch 14/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3807 - accuracy: 0.8245 - mae: 0.2446 - mse: 0.1215 - auc: 0.9081\n",
      "Epoch 15/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3826 - accuracy: 0.8248 - mae: 0.2450 - mse: 0.1216 - auc: 0.9077\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "81\n",
      "81\n",
      "81\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "79\n",
      "78\n",
      "73\n",
      "Epoch 1/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3777 - accuracy: 0.8293 - mae: 0.2424 - mse: 0.1202 - auc: 0.9098\n",
      "Epoch 2/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3752 - accuracy: 0.8303 - mae: 0.2396 - mse: 0.1189 - auc: 0.9113\n",
      "Epoch 3/15\n",
      "302/302 [==============================] - 3s 10ms/step - loss: 0.3739 - accuracy: 0.8303 - mae: 0.2388 - mse: 0.1188 - auc: 0.9118\n",
      "Epoch 4/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3709 - accuracy: 0.8326 - mae: 0.2368 - mse: 0.1178 - auc: 0.9132\n",
      "Epoch 5/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3660 - accuracy: 0.8320 - mae: 0.2339 - mse: 0.1165 - auc: 0.9152\n",
      "Epoch 6/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3640 - accuracy: 0.8354 - mae: 0.2323 - mse: 0.1154 - auc: 0.9166\n",
      "Epoch 7/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3641 - accuracy: 0.8346 - mae: 0.2325 - mse: 0.1154 - auc: 0.9165\n",
      "Epoch 8/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3605 - accuracy: 0.8367 - mae: 0.2297 - mse: 0.1144 - auc: 0.9179\n",
      "Epoch 9/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3574 - accuracy: 0.8388 - mae: 0.2278 - mse: 0.1133 - auc: 0.9195\n",
      "Epoch 10/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3558 - accuracy: 0.8377 - mae: 0.2268 - mse: 0.1130 - auc: 0.9201\n",
      "Epoch 11/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3583 - accuracy: 0.8387 - mae: 0.2284 - mse: 0.1136 - auc: 0.9189\n",
      "Epoch 12/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3608 - accuracy: 0.8349 - mae: 0.2304 - mse: 0.1148 - auc: 0.9176\n",
      "Epoch 13/15\n",
      "302/302 [==============================] - 3s 10ms/step - loss: 0.3546 - accuracy: 0.8393 - mae: 0.2259 - mse: 0.1123 - auc: 0.9209\n",
      "Epoch 14/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3558 - accuracy: 0.8399 - mae: 0.2259 - mse: 0.1129 - auc: 0.9200\n",
      "Epoch 15/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3505 - accuracy: 0.8421 - mae: 0.2231 - mse: 0.1109 - auc: 0.9227\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "Epoch 1/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3468 - accuracy: 0.8438 - mae: 0.2208 - mse: 0.1100 - auc: 0.9240\n",
      "Epoch 2/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3458 - accuracy: 0.8432 - mae: 0.2195 - mse: 0.1095 - auc: 0.9247\n",
      "Epoch 3/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3432 - accuracy: 0.8437 - mae: 0.2196 - mse: 0.1090 - auc: 0.9256\n",
      "Epoch 4/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3426 - accuracy: 0.8449 - mae: 0.2182 - mse: 0.1087 - auc: 0.9260\n",
      "Epoch 5/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3405 - accuracy: 0.8441 - mae: 0.2164 - mse: 0.1082 - auc: 0.9268\n",
      "Epoch 6/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3396 - accuracy: 0.8449 - mae: 0.2166 - mse: 0.1081 - auc: 0.9270\n",
      "Epoch 7/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3382 - accuracy: 0.8443 - mae: 0.2162 - mse: 0.1075 - auc: 0.9277\n",
      "Epoch 8/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3397 - accuracy: 0.8459 - mae: 0.2159 - mse: 0.1075 - auc: 0.9274\n",
      "Epoch 9/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3350 - accuracy: 0.8449 - mae: 0.2134 - mse: 0.1065 - auc: 0.9291\n",
      "Epoch 10/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3413 - accuracy: 0.8458 - mae: 0.2171 - mse: 0.1081 - auc: 0.9266\n",
      "Epoch 11/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3358 - accuracy: 0.8471 - mae: 0.2138 - mse: 0.1066 - auc: 0.9289\n",
      "Epoch 12/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3343 - accuracy: 0.8499 - mae: 0.2131 - mse: 0.1060 - auc: 0.9295\n",
      "Epoch 13/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3337 - accuracy: 0.8472 - mae: 0.2131 - mse: 0.1058 - auc: 0.9299\n",
      "Epoch 14/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3323 - accuracy: 0.8506 - mae: 0.2111 - mse: 0.1052 - auc: 0.9305\n",
      "Epoch 15/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3316 - accuracy: 0.8469 - mae: 0.2112 - mse: 0.1052 - auc: 0.9306\n",
      "85\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "Epoch 1/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3273 - accuracy: 0.8507 - mae: 0.2085 - mse: 0.1041 - auc: 0.9322\n",
      "Epoch 2/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3284 - accuracy: 0.8493 - mae: 0.2087 - mse: 0.1044 - auc: 0.9318\n",
      "Epoch 3/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3264 - accuracy: 0.8520 - mae: 0.2078 - mse: 0.1035 - auc: 0.9329\n",
      "Epoch 4/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3263 - accuracy: 0.8519 - mae: 0.2079 - mse: 0.1034 - auc: 0.9330\n",
      "Epoch 5/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3270 - accuracy: 0.8512 - mae: 0.2083 - mse: 0.1037 - auc: 0.9325\n",
      "Epoch 6/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3246 - accuracy: 0.8505 - mae: 0.2066 - mse: 0.1029 - auc: 0.9336\n",
      "Epoch 7/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3235 - accuracy: 0.8517 - mae: 0.2059 - mse: 0.1028 - auc: 0.9338\n",
      "Epoch 8/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3226 - accuracy: 0.8546 - mae: 0.2048 - mse: 0.1020 - auc: 0.9346\n",
      "Epoch 9/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3184 - accuracy: 0.8567 - mae: 0.2026 - mse: 0.1008 - auc: 0.9360\n",
      "Epoch 10/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3195 - accuracy: 0.8545 - mae: 0.2028 - mse: 0.1010 - auc: 0.9358\n",
      "Epoch 11/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3189 - accuracy: 0.8543 - mae: 0.2023 - mse: 0.1010 - auc: 0.9360\n",
      "Epoch 12/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3207 - accuracy: 0.8558 - mae: 0.2030 - mse: 0.1015 - auc: 0.9352\n",
      "Epoch 13/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3152 - accuracy: 0.8574 - mae: 0.1999 - mse: 0.0995 - auc: 0.9376\n",
      "Epoch 14/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3162 - accuracy: 0.8570 - mae: 0.2006 - mse: 0.1000 - auc: 0.9371\n",
      "Epoch 15/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3149 - accuracy: 0.8586 - mae: 0.2000 - mse: 0.0993 - auc: 0.9379\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "84\n",
      "85\n",
      "Epoch 1/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3136 - accuracy: 0.8590 - mae: 0.1982 - mse: 0.0992 - auc: 0.9381\n",
      "Epoch 2/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3115 - accuracy: 0.8575 - mae: 0.1977 - mse: 0.0986 - auc: 0.9388\n",
      "Epoch 3/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3107 - accuracy: 0.8598 - mae: 0.1966 - mse: 0.0983 - auc: 0.9392\n",
      "Epoch 4/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3096 - accuracy: 0.8594 - mae: 0.1961 - mse: 0.0982 - auc: 0.9395\n",
      "Epoch 5/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3114 - accuracy: 0.8589 - mae: 0.1973 - mse: 0.0985 - auc: 0.9390\n",
      "Epoch 6/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3097 - accuracy: 0.8605 - mae: 0.1959 - mse: 0.0978 - auc: 0.9397\n",
      "Epoch 7/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3073 - accuracy: 0.8616 - mae: 0.1954 - mse: 0.0973 - auc: 0.9405\n",
      "Epoch 8/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3051 - accuracy: 0.8624 - mae: 0.1931 - mse: 0.0964 - auc: 0.9414\n",
      "Epoch 9/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3064 - accuracy: 0.8623 - mae: 0.1944 - mse: 0.0967 - auc: 0.9410\n",
      "Epoch 10/15\n",
      "302/302 [==============================] - 4s 12ms/step - loss: 0.3038 - accuracy: 0.8620 - mae: 0.1930 - mse: 0.0962 - auc: 0.9418\n",
      "Epoch 11/15\n",
      "302/302 [==============================] - 3s 12ms/step - loss: 0.3083 - accuracy: 0.8591 - mae: 0.1951 - mse: 0.0975 - auc: 0.9402\n",
      "Epoch 12/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3108 - accuracy: 0.8590 - mae: 0.1978 - mse: 0.0981 - auc: 0.9396\n",
      "Epoch 13/15\n",
      "302/302 [==============================] - 3s 12ms/step - loss: 0.3085 - accuracy: 0.8608 - mae: 0.1951 - mse: 0.0973 - auc: 0.9402\n",
      "Epoch 14/15\n",
      "302/302 [==============================] - 4s 12ms/step - loss: 0.3064 - accuracy: 0.8615 - mae: 0.1942 - mse: 0.0969 - auc: 0.9410\n",
      "Epoch 15/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3035 - accuracy: 0.8618 - mae: 0.1925 - mse: 0.0963 - auc: 0.9418\n",
      "86\n",
      "86\n",
      "85\n",
      "85\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "Epoch 1/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3099 - accuracy: 0.8592 - mae: 0.1962 - mse: 0.0982 - auc: 0.9394\n",
      "Epoch 2/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3048 - accuracy: 0.8603 - mae: 0.1938 - mse: 0.0965 - auc: 0.9415\n",
      "Epoch 3/15\n",
      "302/302 [==============================] - 3s 12ms/step - loss: 0.2994 - accuracy: 0.8629 - mae: 0.1903 - mse: 0.0948 - auc: 0.9435\n",
      "Epoch 4/15\n",
      "302/302 [==============================] - 4s 12ms/step - loss: 0.3018 - accuracy: 0.8625 - mae: 0.1913 - mse: 0.0955 - auc: 0.9427\n",
      "Epoch 5/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2985 - accuracy: 0.8620 - mae: 0.1892 - mse: 0.0946 - auc: 0.9437\n",
      "Epoch 6/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2978 - accuracy: 0.8657 - mae: 0.1891 - mse: 0.0940 - auc: 0.9443\n",
      "Epoch 7/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3013 - accuracy: 0.8629 - mae: 0.1911 - mse: 0.0953 - auc: 0.9428\n",
      "Epoch 8/15\n",
      "302/302 [==============================] - 4s 12ms/step - loss: 0.2938 - accuracy: 0.8662 - mae: 0.1871 - mse: 0.0929 - auc: 0.9455\n",
      "Epoch 9/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3005 - accuracy: 0.8638 - mae: 0.1896 - mse: 0.0949 - auc: 0.9433\n",
      "Epoch 10/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3000 - accuracy: 0.8635 - mae: 0.1908 - mse: 0.0949 - auc: 0.9434\n",
      "Epoch 11/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2982 - accuracy: 0.8662 - mae: 0.1884 - mse: 0.0940 - auc: 0.9442\n",
      "Epoch 12/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3018 - accuracy: 0.8636 - mae: 0.1910 - mse: 0.0957 - auc: 0.9424\n",
      "Epoch 13/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3025 - accuracy: 0.8642 - mae: 0.1929 - mse: 0.0953 - auc: 0.9427\n",
      "Epoch 14/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3150 - accuracy: 0.8623 - mae: 0.2040 - mse: 0.0993 - auc: 0.9382\n",
      "Epoch 15/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.3066 - accuracy: 0.8643 - mae: 0.1945 - mse: 0.0966 - auc: 0.9411\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "85\n",
      "Epoch 1/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2979 - accuracy: 0.8671 - mae: 0.1892 - mse: 0.0940 - auc: 0.9440\n",
      "Epoch 2/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2929 - accuracy: 0.8696 - mae: 0.1858 - mse: 0.0921 - auc: 0.9462\n",
      "Epoch 3/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2939 - accuracy: 0.8669 - mae: 0.1869 - mse: 0.0926 - auc: 0.9456\n",
      "Epoch 4/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2921 - accuracy: 0.8678 - mae: 0.1854 - mse: 0.0924 - auc: 0.9462\n",
      "Epoch 5/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2939 - accuracy: 0.8658 - mae: 0.1862 - mse: 0.0931 - auc: 0.9454\n",
      "Epoch 6/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2878 - accuracy: 0.8689 - mae: 0.1837 - mse: 0.0912 - auc: 0.9476\n",
      "Epoch 7/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2851 - accuracy: 0.8717 - mae: 0.1809 - mse: 0.0901 - auc: 0.9487\n",
      "Epoch 8/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2909 - accuracy: 0.8677 - mae: 0.1850 - mse: 0.0922 - auc: 0.9464\n",
      "Epoch 9/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2889 - accuracy: 0.8687 - mae: 0.1834 - mse: 0.0915 - auc: 0.9474\n",
      "Epoch 10/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2908 - accuracy: 0.8673 - mae: 0.1841 - mse: 0.0917 - auc: 0.9469\n",
      "Epoch 11/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2882 - accuracy: 0.8711 - mae: 0.1827 - mse: 0.0907 - auc: 0.9479\n",
      "Epoch 12/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2856 - accuracy: 0.8713 - mae: 0.1808 - mse: 0.0902 - auc: 0.9487\n",
      "Epoch 13/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2858 - accuracy: 0.8694 - mae: 0.1811 - mse: 0.0904 - auc: 0.9484\n",
      "Epoch 14/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2844 - accuracy: 0.8706 - mae: 0.1801 - mse: 0.0901 - auc: 0.9489\n",
      "Epoch 15/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2858 - accuracy: 0.8725 - mae: 0.1813 - mse: 0.0902 - auc: 0.9485\n",
      "87\n",
      "86\n",
      "87\n",
      "87\n",
      "86\n",
      "86\n",
      "86\n",
      "87\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "Epoch 1/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2856 - accuracy: 0.8712 - mae: 0.1808 - mse: 0.0903 - auc: 0.9485\n",
      "Epoch 2/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2828 - accuracy: 0.8716 - mae: 0.1797 - mse: 0.0898 - auc: 0.9494\n",
      "Epoch 3/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2839 - accuracy: 0.8700 - mae: 0.1806 - mse: 0.0899 - auc: 0.9492\n",
      "Epoch 4/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2811 - accuracy: 0.8722 - mae: 0.1784 - mse: 0.0891 - auc: 0.9501\n",
      "Epoch 5/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2866 - accuracy: 0.8704 - mae: 0.1823 - mse: 0.0907 - auc: 0.9482\n",
      "Epoch 6/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2801 - accuracy: 0.8740 - mae: 0.1771 - mse: 0.0883 - auc: 0.9506\n",
      "Epoch 7/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2816 - accuracy: 0.8727 - mae: 0.1789 - mse: 0.0891 - auc: 0.9500\n",
      "Epoch 8/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2806 - accuracy: 0.8752 - mae: 0.1781 - mse: 0.0884 - auc: 0.9505\n",
      "Epoch 9/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2813 - accuracy: 0.8732 - mae: 0.1785 - mse: 0.0888 - auc: 0.9502\n",
      "Epoch 10/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2825 - accuracy: 0.8738 - mae: 0.1794 - mse: 0.0892 - auc: 0.9497\n",
      "Epoch 11/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2800 - accuracy: 0.8748 - mae: 0.1767 - mse: 0.0883 - auc: 0.9507\n",
      "Epoch 12/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2779 - accuracy: 0.8724 - mae: 0.1756 - mse: 0.0882 - auc: 0.9511\n",
      "Epoch 13/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2804 - accuracy: 0.8744 - mae: 0.1767 - mse: 0.0881 - auc: 0.9507\n",
      "Epoch 14/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2798 - accuracy: 0.8737 - mae: 0.1784 - mse: 0.0885 - auc: 0.9507\n",
      "Epoch 15/15\n",
      "302/302 [==============================] - 3s 11ms/step - loss: 0.2801 - accuracy: 0.8748 - mae: 0.1771 - mse: 0.0883 - auc: 0.9506\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "Epoch_S = 15\n",
    "\n",
    "l, r , lbls = data_generator(ds_train)\n",
    "\n",
    "l = np.array(l).reshape(-1,1024,1)\n",
    "r = np.array(r).reshape(-1,512,1)\n",
    "lbls=np.array(lbls)\n",
    "\n",
    "history = History()\n",
    "\n",
    "siamese_net = siamese_model_attentiveFp_sider()\n",
    "\n",
    "\n",
    "s = siamese_net.fit([l, r], lbls, epochs = Epoch_S, shuffle=True, batch_size=128, callbacks=[history])\n",
    "\n",
    "for j in range(1000):\n",
    "    C=1\n",
    "    Before = int(s.history['accuracy'][-1]*100)\n",
    "    for i in range(2,Epoch_S+1):\n",
    "        if  int(s.history['accuracy'][-i]*100)== Before:\n",
    "            C=C+1\n",
    "        else:\n",
    "            C=1\n",
    "        Before=int(s.history['accuracy'][-i]*100)\n",
    "        print(Before)\n",
    "    if C==Epoch_S:\n",
    "        break\n",
    "    s = siamese_net.fit([l, r], lbls, epochs = Epoch_S, shuffle=True, batch_size=128, callbacks=history)\n",
    "print(j+1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "LK16tu418EV4"
   },
   "source": [
    "### Model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "SZchDfSt8EV5"
   },
   "outputs": [],
   "source": [
    "valid_ds = {}\n",
    "\n",
    "for i, task in enumerate(sider_tasks):\n",
    "    temp = []\n",
    "    for j , data in enumerate(ds_study):\n",
    "        smiles, embbed_drug, onehot_task, embbed_task, lbl, task_name = data\n",
    "        if task ==  task_name:\n",
    "            temp.append(data)\n",
    "\n",
    "    valid_ds[task] = temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "MZ1fNY7H8EV5",
    "run_control": {
     "marked": true
    }
   },
   "outputs": [],
   "source": [
    "task_scores = [sider_tasks for sider_tasks in range(len(sider_tasks))]\n",
    "\n",
    "for i, task in enumerate(sider_tasks):\n",
    "\n",
    "    l_val = []\n",
    "    r_val = []\n",
    "    lbls_valid = []\n",
    "    for data in valid_ds[task]:\n",
    "\n",
    "        smiles, embbed_drug, onehot_task, embbed_task, lbl, task_name = data\n",
    "        l_val.append(embbed_drug[0])\n",
    "        r_val.append(embbed_task)\n",
    "        lbls_valid.append(lbl)\n",
    "\n",
    "    l1 = np.array(l_val)\n",
    "    r1 = np.array(r_val)\n",
    "    lbls_valid = np.array(lbls_valid)\n",
    "\n",
    "    y_pred = siamese_net.predict([l1,r1])\n",
    "\n",
    "    result = (y_pred)\n",
    "    task_scores[i] = task, result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "bW6KCg-98EV5",
    "outputId": "ab05c8a5-184a-4258-f867-a01100066c7d",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " --------------------------------- \n",
      "Hepatobiliary disorders:\n",
      "1- Vancomycin: [0.350539]\n",
      "2- cyclosporine: [0.9355659]\n",
      "3- fingolimod: [0.56871724]\n",
      "4- interferon-beta 1a: [0.11978364]\n",
      "5- Remdesivir: [0.828961]\n",
      "6- Oseltamivir: [0.37015772]\n",
      "7- Ritonavir : [0.98959655]\n",
      "8- Molnupiravir : [0.4740463]\n",
      "9- Paxlovid : [0.30573225]\n",
      "10- Baxdrostat: [0.20183662]\n",
      "11- Guanfacine : [0.202806]\n",
      "12- Liraglutide: [0.10674441]\n",
      "13- Linagliptin: [0.15892702]\n",
      " --------------------------------- \n",
      "Metabolism and nutrition disorders:\n",
      "1- Vancomycin: [0.6682925]\n",
      "2- cyclosporine: [0.9988003]\n",
      "3- fingolimod: [0.68529713]\n",
      "4- interferon-beta 1a: [0.44358477]\n",
      "5- Remdesivir: [0.81775]\n",
      "6- Oseltamivir: [0.6768574]\n",
      "7- Ritonavir : [0.9990827]\n",
      "8- Molnupiravir : [0.7079015]\n",
      "9- Paxlovid : [0.7342775]\n",
      "10- Baxdrostat: [0.4607547]\n",
      "11- Guanfacine : [0.7755146]\n",
      "12- Liraglutide: [0.2249364]\n",
      "13- Linagliptin: [0.41168305]\n",
      " --------------------------------- \n",
      "Product issues:\n",
      "1- Vancomycin: [0.11146241]\n",
      "2- cyclosporine: [0.137191]\n",
      "3- fingolimod: [0.13704643]\n",
      "4- interferon-beta 1a: [0.114995]\n",
      "5- Remdesivir: [0.11988485]\n",
      "6- Oseltamivir: [0.1512165]\n",
      "7- Ritonavir : [0.10556015]\n",
      "8- Molnupiravir : [0.11072245]\n",
      "9- Paxlovid : [0.1474882]\n",
      "10- Baxdrostat: [0.11162975]\n",
      "11- Guanfacine : [0.13727602]\n",
      "12- Liraglutide: [0.12576103]\n",
      "13- Linagliptin: [0.09925777]\n",
      " --------------------------------- \n",
      "Eye disorders:\n",
      "1- Vancomycin: [0.37316865]\n",
      "2- cyclosporine: [0.9968556]\n",
      "3- fingolimod: [0.58562964]\n",
      "4- interferon-beta 1a: [0.35304356]\n",
      "5- Remdesivir: [0.46498215]\n",
      "6- Oseltamivir: [0.7521177]\n",
      "7- Ritonavir : [0.99550885]\n",
      "8- Molnupiravir : [0.2601191]\n",
      "9- Paxlovid : [0.8024298]\n",
      "10- Baxdrostat: [0.31855094]\n",
      "11- Guanfacine : [0.87677103]\n",
      "12- Liraglutide: [0.30758983]\n",
      "13- Linagliptin: [0.3048868]\n",
      " --------------------------------- \n",
      "Investigations:\n",
      "1- Vancomycin: [0.90351677]\n",
      "2- cyclosporine: [0.99943775]\n",
      "3- fingolimod: [0.88745964]\n",
      "4- interferon-beta 1a: [0.5810516]\n",
      "5- Remdesivir: [0.9449169]\n",
      "6- Oseltamivir: [0.91552603]\n",
      "7- Ritonavir : [0.9997351]\n",
      "8- Molnupiravir : [0.92322636]\n",
      "9- Paxlovid : [0.93050575]\n",
      "10- Baxdrostat: [0.4733318]\n",
      "11- Guanfacine : [0.92541873]\n",
      "12- Liraglutide: [0.31370604]\n",
      "13- Linagliptin: [0.4264763]\n",
      " --------------------------------- \n",
      "Musculoskeletal and connective tissue disorders:\n",
      "1- Vancomycin: [0.63208586]\n",
      "2- cyclosporine: [0.99901724]\n",
      "3- fingolimod: [0.6631824]\n",
      "4- interferon-beta 1a: [0.5274877]\n",
      "5- Remdesivir: [0.6554395]\n",
      "6- Oseltamivir: [0.8189204]\n",
      "7- Ritonavir : [0.9988821]\n",
      "8- Molnupiravir : [0.6209959]\n",
      "9- Paxlovid : [0.86491036]\n",
      "10- Baxdrostat: [0.4607547]\n",
      "11- Guanfacine : [0.9015927]\n",
      "12- Liraglutide: [0.21453649]\n",
      "13- Linagliptin: [0.46272138]\n",
      " --------------------------------- \n",
      "Gastrointestinal disorders:\n",
      "1- Vancomycin: [0.98184705]\n",
      "2- cyclosporine: [0.99994266]\n",
      "3- fingolimod: [0.983924]\n",
      "4- interferon-beta 1a: [0.7985892]\n",
      "5- Remdesivir: [0.98611283]\n",
      "6- Oseltamivir: [0.99220824]\n",
      "7- Ritonavir : [0.99994934]\n",
      "8- Molnupiravir : [0.98091745]\n",
      "9- Paxlovid : [0.9941251]\n",
      "10- Baxdrostat: [0.6855211]\n",
      "11- Guanfacine : [0.9946332]\n",
      "12- Liraglutide: [0.5868407]\n",
      "13- Linagliptin: [0.63066363]\n",
      " --------------------------------- \n",
      "Social circumstances:\n",
      "1- Vancomycin: [0.11000958]\n",
      "2- cyclosporine: [0.5133571]\n",
      "3- fingolimod: [0.25951332]\n",
      "4- interferon-beta 1a: [0.10947442]\n",
      "5- Remdesivir: [0.20011988]\n",
      "6- Oseltamivir: [0.30747277]\n",
      "7- Ritonavir : [0.29430693]\n",
      "8- Molnupiravir : [0.10992336]\n",
      "9- Paxlovid : [0.30070534]\n",
      "10- Baxdrostat: [0.10912281]\n",
      "11- Guanfacine : [0.28574395]\n",
      "12- Liraglutide: [0.14034581]\n",
      "13- Linagliptin: [0.04741859]\n",
      " --------------------------------- \n",
      "Immune system disorders:\n",
      "1- Vancomycin: [0.64423734]\n",
      "2- cyclosporine: [0.99841356]\n",
      "3- fingolimod: [0.77885944]\n",
      "4- interferon-beta 1a: [0.45008206]\n",
      "5- Remdesivir: [0.66411567]\n",
      "6- Oseltamivir: [0.75834095]\n",
      "7- Ritonavir : [0.9945158]\n",
      "8- Molnupiravir : [0.69150203]\n",
      "9- Paxlovid : [0.82830906]\n",
      "10- Baxdrostat: [0.3742125]\n",
      "11- Guanfacine : [0.7459092]\n",
      "12- Liraglutide: [0.42719823]\n",
      "13- Linagliptin: [0.36402112]\n",
      " --------------------------------- \n",
      "Reproductive system and breast disorders:\n",
      "1- Vancomycin: [0.25330216]\n",
      "2- cyclosporine: [0.9860876]\n",
      "3- fingolimod: [0.6082279]\n",
      "4- interferon-beta 1a: [0.20653781]\n",
      "5- Remdesivir: [0.5904837]\n",
      "6- Oseltamivir: [0.5158926]\n",
      "7- Ritonavir : [0.98843944]\n",
      "8- Molnupiravir : [0.32470337]\n",
      "9- Paxlovid : [0.51981866]\n",
      "10- Baxdrostat: [0.24280572]\n",
      "11- Guanfacine : [0.41353092]\n",
      "12- Liraglutide: [0.20492667]\n",
      "13- Linagliptin: [0.15685192]\n",
      " --------------------------------- \n",
      "Neoplasms benign, malignant and unspecified (incl cysts and polyps):\n",
      "1- Vancomycin: [0.19139883]\n",
      "2- cyclosporine: [0.5936509]\n",
      "3- fingolimod: [0.4607547]\n",
      "4- interferon-beta 1a: [0.13758257]\n",
      "5- Remdesivir: [0.34895402]\n",
      "6- Oseltamivir: [0.4055559]\n",
      "7- Ritonavir : [0.48422453]\n",
      "8- Molnupiravir : [0.24559411]\n",
      "9- Paxlovid : [0.3663663]\n",
      "10- Baxdrostat: [0.13966161]\n",
      "11- Guanfacine : [0.2509421]\n",
      "12- Liraglutide: [0.21375227]\n",
      "13- Linagliptin: [0.07174683]\n",
      " --------------------------------- \n",
      "General disorders and administration site conditions:\n",
      "1- Vancomycin: [0.9517914]\n",
      "2- cyclosporine: [0.99996454]\n",
      "3- fingolimod: [0.96249115]\n",
      "4- interferon-beta 1a: [0.7632922]\n",
      "5- Remdesivir: [0.9456134]\n",
      "6- Oseltamivir: [0.9835354]\n",
      "7- Ritonavir : [0.9992969]\n",
      "8- Molnupiravir : [0.9430547]\n",
      "9- Paxlovid : [0.9879329]\n",
      "10- Baxdrostat: [0.6502152]\n",
      "11- Guanfacine : [0.99042165]\n",
      "12- Liraglutide: [0.6255477]\n",
      "13- Linagliptin: [0.6160714]\n",
      " --------------------------------- \n",
      "Endocrine disorders:\n",
      "1- Vancomycin: [0.1642527]\n",
      "2- cyclosporine: [0.56138146]\n",
      "3- fingolimod: [0.4161331]\n",
      "4- interferon-beta 1a: [0.1215297]\n",
      "5- Remdesivir: [0.31331074]\n",
      "6- Oseltamivir: [0.37867832]\n",
      "7- Ritonavir : [0.3481509]\n",
      "8- Molnupiravir : [0.20800105]\n",
      "9- Paxlovid : [0.3477806]\n",
      "10- Baxdrostat: [0.11399499]\n",
      "11- Guanfacine : [0.23999429]\n",
      "12- Liraglutide: [0.20412207]\n",
      "13- Linagliptin: [0.06077212]\n",
      " --------------------------------- \n",
      "Surgical and medical procedures:\n",
      "1- Vancomycin: [0.18713537]\n",
      "2- cyclosporine: [0.3462767]\n",
      "3- fingolimod: [0.26013345]\n",
      "4- interferon-beta 1a: [0.21425813]\n",
      "5- Remdesivir: [0.22242573]\n",
      "6- Oseltamivir: [0.28801316]\n",
      "7- Ritonavir : [0.26346934]\n",
      "8- Molnupiravir : [0.1871765]\n",
      "9- Paxlovid : [0.2833307]\n",
      "10- Baxdrostat: [0.18995273]\n",
      "11- Guanfacine : [0.2824551]\n",
      "12- Liraglutide: [0.20408422]\n",
      "13- Linagliptin: [0.10117742]\n",
      " --------------------------------- \n",
      "Vascular disorders:\n",
      "1- Vancomycin: [0.7547036]\n",
      "2- cyclosporine: [0.99910265]\n",
      "3- fingolimod: [0.78421855]\n",
      "4- interferon-beta 1a: [0.63703895]\n",
      "5- Remdesivir: [0.7291052]\n",
      "6- Oseltamivir: [0.90727437]\n",
      "7- Ritonavir : [0.99891865]\n",
      "8- Molnupiravir : [0.7132642]\n",
      "9- Paxlovid : [0.9336145]\n",
      "10- Baxdrostat: [0.51019704]\n",
      "11- Guanfacine : [0.9535829]\n",
      "12- Liraglutide: [0.312742]\n",
      "13- Linagliptin: [0.4607547]\n",
      " --------------------------------- \n",
      "Blood and lymphatic system disorders:\n",
      "1- Vancomycin: [0.6117393]\n",
      "2- cyclosporine: [0.9871889]\n",
      "3- fingolimod: [0.6924893]\n",
      "4- interferon-beta 1a: [0.21234566]\n",
      "5- Remdesivir: [0.9077817]\n",
      "6- Oseltamivir: [0.5214833]\n",
      "7- Ritonavir : [0.99821585]\n",
      "8- Molnupiravir : [0.73013943]\n",
      "9- Paxlovid : [0.4707032]\n",
      "10- Baxdrostat: [0.28509998]\n",
      "11- Guanfacine : [0.29770654]\n",
      "12- Liraglutide: [0.16030711]\n",
      "13- Linagliptin: [0.2301569]\n",
      " --------------------------------- \n",
      "Skin and subcutaneous tissue disorders:\n",
      "1- Vancomycin: [0.9555234]\n",
      "2- cyclosporine: [0.9999765]\n",
      "3- fingolimod: [0.97125864]\n",
      "4- interferon-beta 1a: [0.7465097]\n",
      "5- Remdesivir: [0.96158326]\n",
      "6- Oseltamivir: [0.98291904]\n",
      "7- Ritonavir : [0.9992157]\n",
      "8- Molnupiravir : [0.95593816]\n",
      "9- Paxlovid : [0.9874487]\n",
      "10- Baxdrostat: [0.6458949]\n",
      "11- Guanfacine : [0.9873192]\n",
      "12- Liraglutide: [0.6408335]\n",
      "13- Linagliptin: [0.6234473]\n",
      " --------------------------------- \n",
      "Congenital, familial and genetic disorders:\n",
      "1- Vancomycin: [0.21474612]\n",
      "2- cyclosporine: [0.40555605]\n",
      "3- fingolimod: [0.32429162]\n",
      "4- interferon-beta 1a: [0.20023572]\n",
      "5- Remdesivir: [0.28285944]\n",
      "6- Oseltamivir: [0.3198192]\n",
      "7- Ritonavir : [0.27123255]\n",
      "8- Molnupiravir : [0.22801375]\n",
      "9- Paxlovid : [0.31307882]\n",
      "10- Baxdrostat: [0.17902952]\n",
      "11- Guanfacine : [0.26686]\n",
      "12- Liraglutide: [0.22741392]\n",
      "13- Linagliptin: [0.09290808]\n",
      " --------------------------------- \n",
      "Infections and infestations:\n",
      "1- Vancomycin: [0.7209294]\n",
      "2- cyclosporine: [0.9990195]\n",
      "3- fingolimod: [0.86128265]\n",
      "4- interferon-beta 1a: [0.39893284]\n",
      "5- Remdesivir: [0.7338598]\n",
      "6- Oseltamivir: [0.8307425]\n",
      "7- Ritonavir : [0.9940871]\n",
      "8- Molnupiravir : [0.7701676]\n",
      "9- Paxlovid : [0.8894935]\n",
      "10- Baxdrostat: [0.35178015]\n",
      "11- Guanfacine : [0.8304193]\n",
      "12- Liraglutide: [0.38965222]\n",
      "13- Linagliptin: [0.34117898]\n",
      " --------------------------------- \n",
      "Respiratory, thoracic and mediastinal disorders:\n",
      "1- Vancomycin: [0.70925397]\n",
      "2- cyclosporine: [0.99891233]\n",
      "3- fingolimod: [0.7379136]\n",
      "4- interferon-beta 1a: [0.62246764]\n",
      "5- Remdesivir: [0.67769533]\n",
      "6- Oseltamivir: [0.88113624]\n",
      "7- Ritonavir : [0.9986583]\n",
      "8- Molnupiravir : [0.6682426]\n",
      "9- Paxlovid : [0.9143398]\n",
      "10- Baxdrostat: [0.4607547]\n",
      "11- Guanfacine : [0.941073]\n",
      "12- Liraglutide: [0.27824336]\n",
      "13- Linagliptin: [0.4607547]\n",
      " --------------------------------- \n",
      "Psychiatric disorders:\n",
      "1- Vancomycin: [0.6789369]\n",
      "2- cyclosporine: [0.998044]\n",
      "3- fingolimod: [0.6719494]\n",
      "4- interferon-beta 1a: [0.5631037]\n",
      "5- Remdesivir: [0.6724399]\n",
      "6- Oseltamivir: [0.8257878]\n",
      "7- Ritonavir : [0.9985529]\n",
      "8- Molnupiravir : [0.63636094]\n",
      "9- Paxlovid : [0.8624155]\n",
      "10- Baxdrostat: [0.44196203]\n",
      "11- Guanfacine : [0.9217546]\n",
      "12- Liraglutide: [0.19605997]\n",
      "13- Linagliptin: [0.3840042]\n",
      " --------------------------------- \n",
      "Renal and urinary disorders:\n",
      "1- Vancomycin: [0.6444223]\n",
      "2- cyclosporine: [0.9979081]\n",
      "3- fingolimod: [0.6047732]\n",
      "4- interferon-beta 1a: [0.30830526]\n",
      "5- Remdesivir: [0.7780845]\n",
      "6- Oseltamivir: [0.59163964]\n",
      "7- Ritonavir : [0.9984401]\n",
      "8- Molnupiravir : [0.669031]\n",
      "9- Paxlovid : [0.63216066]\n",
      "10- Baxdrostat: [0.33587766]\n",
      "11- Guanfacine : [0.6933416]\n",
      "12- Liraglutide: [0.18669999]\n",
      "13- Linagliptin: [0.28938282]\n",
      " --------------------------------- \n",
      "Pregnancy, puerperium and perinatal conditions:\n",
      "1- Vancomycin: [0.14845803]\n",
      "2- cyclosporine: [0.31298214]\n",
      "3- fingolimod: [0.23485419]\n",
      "4- interferon-beta 1a: [0.18016148]\n",
      "5- Remdesivir: [0.1809606]\n",
      "6- Oseltamivir: [0.26286945]\n",
      "7- Ritonavir : [0.19785595]\n",
      "8- Molnupiravir : [0.14919192]\n",
      "9- Paxlovid : [0.25825995]\n",
      "10- Baxdrostat: [0.16041315]\n",
      "11- Guanfacine : [0.24090257]\n",
      "12- Liraglutide: [0.21106815]\n",
      "13- Linagliptin: [0.09774888]\n",
      " --------------------------------- \n",
      "Ear and labyrinth disorders:\n",
      "1- Vancomycin: [0.24334621]\n",
      "2- cyclosporine: [0.96135956]\n",
      "3- fingolimod: [0.31520635]\n",
      "4- interferon-beta 1a: [0.24876356]\n",
      "5- Remdesivir: [0.4802073]\n",
      "6- Oseltamivir: [0.47925326]\n",
      "7- Ritonavir : [0.96433866]\n",
      "8- Molnupiravir : [0.22118849]\n",
      "9- Paxlovid : [0.6170323]\n",
      "10- Baxdrostat: [0.29607743]\n",
      "11- Guanfacine : [0.6565564]\n",
      "12- Liraglutide: [0.16052583]\n",
      "13- Linagliptin: [0.20782629]\n",
      " --------------------------------- \n",
      "Cardiac disorders:\n",
      "1- Vancomycin: [0.65778995]\n",
      "2- cyclosporine: [0.9964801]\n",
      "3- fingolimod: [0.6836935]\n",
      "4- interferon-beta 1a: [0.5540538]\n",
      "5- Remdesivir: [0.6289433]\n",
      "6- Oseltamivir: [0.8171531]\n",
      "7- Ritonavir : [0.9965235]\n",
      "8- Molnupiravir : [0.54102266]\n",
      "9- Paxlovid : [0.8611581]\n",
      "10- Baxdrostat: [0.33134896]\n",
      "11- Guanfacine : [0.9144474]\n",
      "12- Liraglutide: [0.24835905]\n",
      "13- Linagliptin: [0.34225714]\n",
      " --------------------------------- \n",
      "Nervous system disorders:\n",
      "1- Vancomycin: [0.9748478]\n",
      "2- cyclosporine: [0.9999695]\n",
      "3- fingolimod: [0.9791769]\n",
      "4- interferon-beta 1a: [0.85916173]\n",
      "5- Remdesivir: [0.97229624]\n",
      "6- Oseltamivir: [0.99133545]\n",
      "7- Ritonavir : [0.9997995]\n",
      "8- Molnupiravir : [0.96863556]\n",
      "9- Paxlovid : [0.9936583]\n",
      "10- Baxdrostat: [0.71916264]\n",
      "11- Guanfacine : [0.9954692]\n",
      "12- Liraglutide: [0.6698999]\n",
      "13- Linagliptin: [0.6761001]\n",
      " --------------------------------- \n",
      "Injury, poisoning and procedural complications:\n",
      "1- Vancomycin: [0.67323774]\n",
      "2- cyclosporine: [0.9984876]\n",
      "3- fingolimod: [0.7830428]\n",
      "4- interferon-beta 1a: [0.36187458]\n",
      "5- Remdesivir: [0.6967679]\n",
      "6- Oseltamivir: [0.76587427]\n",
      "7- Ritonavir : [0.9969555]\n",
      "8- Molnupiravir : [0.714399]\n",
      "9- Paxlovid : [0.8287672]\n",
      "10- Baxdrostat: [0.36459988]\n",
      "11- Guanfacine : [0.7833328]\n",
      "12- Liraglutide: [0.26521188]\n",
      "13- Linagliptin: [0.36188668]\n"
     ]
    }
   ],
   "source": [
    "for task in task_scores:\n",
    "    print(\" --------------------------------- \")\n",
    "    print(F'{task[0]}:')\n",
    "    for i, drug in enumerate(task[1]):\n",
    "        print(F'{i+1}- {drug_name[i]}: {drug}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "fUZ2rO__8EV5"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "zIKQi__XAcia"
   ],
   "provenance": []
  },
  "gpuClass": "standard",
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
